def init_search_space(self):
search_space = SearchSpace(
parameters=[
RangeParameter(name="cov_para_1", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
RangeParameter(name="cov_para_2", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
RangeParameter(name="cov_para_3", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
RangeParameter(name="cov_para_4", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
# RangeParameter(name="cov_para_5", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
# RangeParameter(name="cov_para_6", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
RangeParameter(name="beta_1a", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_1b", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_2a", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_2b", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_3a", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_3b", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_4a", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_4b", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_5a", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_5b", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="lambda_expon_1", parameter_type=ParameterType.FLOAT, lower=0.001, upper=0.1),
RangeParameter(name="lambda_expon_2", parameter_type=ParameterType.FLOAT, lower=0.001, upper=0.1),
RangeParameter(name="lambda_expon_3", parameter_type=ParameterType.FLOAT, lower=0.001, upper=0.1),
]
)
self.exp = SimpleExperiment(
name="0",
search_space=search_space,
evaluation_function=self.evaluate_parameter,
objective_name="adj_return"
)
class optimization():
def __init__(self):
self.max_worker = multiprocessing.cpu_count()
if self.max_worker > 1:
self.max_worker -= 5
#self.max_worker = 1
self.best_obj_so_far = float('-inf')
self.exp = None #define an experiment for ax
self.best_parameters_from_experiment = None
self.global_iteration_count = 0
def evaluate_parameter(self,parameter, weight=None):
total_returns = []
q = multiprocessing.Queue(maxsize = self.max_worker)
for iteration_index in range(0,C.ROUNDS_OF_WORKER):
p_list = []
for worker in range(0,self.max_worker):
try:
p = multiprocessing.Process(target = self.get_one_sample_score,\
args = (q,parameter))
p.start()
p_list.append(p)
except:
pass
for j in range(len(p_list)):
res = q.get()
total_returns.append(res[0])
self.global_iteration_count += 1
if np.mean(total_returns)>self.best_obj_so_far:
self.best_obj_so_far = np.mean(total_returns)
self.best_parameters_from_experiment = parameter
print('the best objective so far is',self.best_obj_so_far)
print('the associated best parameters are',parameter)
print('the current time is',datetime.datetime.now())
cwd = os.getcwd()
parameter_file = 'best_parameter_from_direct_experiment.json'
cwd = os.path.join(cwd,parameter_file)
with open(cwd, 'w') as statusFile:
statusFile.write(jsonpickle.encode(self.best_parameters_from_experiment))
print('this is global iteration',self.global_iteration_count)
print('the objective for ths iteration is',np.mean(total_returns))
print('the current time is',datetime.datetime.now())
print(' ')
optimization_objective = (np.mean(total_returns) - np.std(total_returns) * 0.5, np.std(total_returns))
return {'adj_return': optimization_objective}
def get_one_sample_score(self,q,free_parameters):
free_para1 = free_parameters['cov_para_1']
free_para2 = free_parameters['cov_para_2']
free_para3 = free_parameters['cov_para_3']
free_para4 = free_parameters['cov_para_4']
free_para5 = free_parameters['cov_para_5']
free_para6 = free_parameters['cov_para_6']
free_para7 = free_parameters['cov_para_7']
free_para8 = free_parameters['cov_para_8']
free_para9 = free_parameters['cov_para_9']
free_para10 = free_parameters['cov_para_10']
diag_1 = 1
diag_2 = np.sqrt(1-free_para1**2)
if (1-free_para2**2-free_para3**2)<0:
diag_3 = np.sqrt(np.abs(1-free_para2**2-free_para3**2))
else:
diag_3 = np.sqrt(1-free_para2**2-free_para3**2)
if (1-free_para4**2-free_para5**2-free_para6**2)<0:
diag_4 = np.sqrt(np.abs(1-free_para4**2-free_para5**2-free_para6**2))
else:
diag_4 = np.sqrt(1-free_para4**2-free_para5**2-free_para6**2)
if (1-free_para7**2-free_para8**2-free_para9**2-free_para10**2)<0:
diag_5 = np.sqrt(np.abs(1-free_para7**2-free_para8**2-free_para9**2-free_para10**2))
else:
diag_5 = np.sqrt(1-free_para7**2-free_para8**2-free_para9**2-free_para10**2)
lower_triangular_matrix = np.asarray([[diag_1,0,0,0,0],
[free_para1,diag_2,0,0,0],
[free_para2,free_para3,diag_3,0,0],
[free_para4,free_para5,free_para6,diag_4,0],
[free_para7,free_para8,free_para9,free_para10,diag_5]])
cov_matrix = lower_triangular_matrix@lower_triangular_matrix.transpose()
#now, define the marginal distribution of the gaussian copula
univerates = [{'loc': 0,
'scale': 1,
'a': free_parameters['beta_1a'],
'b': free_parameters['beta_1b'],
'type': 'copulas.univariate.beta.BetaUnivariate'},
{'loc': 0,
'scale': 1,
'a': free_parameters['beta_2a'],
'b': free_parameters['beta_2b'],
'type': 'copulas.univariate.beta.BetaUnivariate'},
{'loc': 0,
'scale': 1,
'a': free_parameters['beta_3a'],
'b': free_parameters['beta_3b'],
'type': 'copulas.univariate.beta.BetaUnivariate'},
{'loc': 0,
'scale': 1,
'a': free_parameters['beta_4a'],
'b': free_parameters['beta_4b'],
'type': 'copulas.univariate.beta.BetaUnivariate'},
{'loc': 0,
'scale': 1,
'a': free_parameters['beta_5a'],
'b': free_parameters['beta_5b'],
'type': 'copulas.univariate.beta.BetaUnivariate'}]
#now, we construct the gaussian copula
copula_parameters = {}
copula_parameters['covariance'] = cov_matrix
copula_parameters['univariates'] = univerates
copula_parameters['type'] = 'copulas.multivariate.gaussian.GaussianMultivariate'
copula_parameters['columns'] = [0,1,2,3,4]
new_dist = Multivariate.from_dict(copula_parameters)
#other parameters needed for transforming the features
lambda_expon_1 = free_parameters['lambda_expon_1']
lambda_expon_2 = free_parameters['lambda_expon_2']
lambda_expon_3 = free_parameters['lambda_expon_3']
lambda_expon_4 = free_parameters['lambda_expon_4']
lambda_expons = [lambda_expon_1,lambda_expon_2,lambda_expon_3,lambda_expon_4]
#now, we begin to simulate trading
#first, initialize the observation
locol_env = trading_vix.trading_vix()
this_trajectory_reward = []
has_at_least_sell = False
null_objective = True
current_feature = locol_env.reset()
for time_index in range(0,200):
#compute an action given current observation
transformed_features = []
for feature_index in range(len(lambda_expons)):
transformation = expon.cdf(current_feature[feature_index,0],scale = 1.0/lambda_expons[feature_index])
min_transformation = 0.1
transformation = min_transformation*np.exp(np.log(1.0/min_transformation)*transformation)
transformed_features.append(transformation)
transformed_features = np.asarray(transformed_features)
transformed_features = np.reshape(transformed_features,(1,-1))
#holding_position = expit(current_feature[-1,:][0])
holding_position = current_feature[-1,:][0]
if holding_position<0:
print('holding is less than 0, there is some problem and the holding position is',holding_position)
if holding_position>1:
print('holding is greater than 1, there is some problem and the holding position is',holding_position)
min_transformed_holding = 0.1
transformed_holding = min_transformed_holding*np.exp(np.log(1.0/min_transformed_holding)*holding_position)
transformed_holding = np.reshape(transformed_holding,(1,1))
data_point_for_df = np.concatenate((transformed_features,transformed_holding),axis = 1)
assert data_point_for_df.shape[1] == 5
data_point_for_copula = pd.DataFrame(data_point_for_df)
action = new_dist.cdf(data_point_for_copula)
#print('action in optimization trading vix is',action)
#apply the action to the environment
current_feature, reward, has_at_least_sell = locol_env.step(action)
if has_at_least_sell and null_objective:
#print('sold at least once')
null_objective = False
#record reward
this_trajectory_reward.append(reward)
#final time step,get the long term reward
reward = locol_env.final()
this_trajectory_reward.append(reward)
if null_objective:
objective = -1e9
else:
objective = np.sum(this_trajectory_reward)
q.put([objective])
def init_search_space(self):
search_space = SearchSpace(
parameters=[
RangeParameter(name="cov_para_1", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
RangeParameter(name="cov_para_2", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
RangeParameter(name="cov_para_3", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
RangeParameter(name="cov_para_4", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
RangeParameter(name="cov_para_5", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
RangeParameter(name="cov_para_6", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
RangeParameter(name="cov_para_7", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
RangeParameter(name="cov_para_8", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
RangeParameter(name="cov_para_9", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
RangeParameter(name="cov_para_10", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
RangeParameter(name="beta_1a", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_1b", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_2a", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_2b", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_3a", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_3b", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_4a", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_4b", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_5a", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_5b", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="lambda_expon_1", parameter_type=ParameterType.FLOAT, lower=0.001, upper=0.1),
RangeParameter(name="lambda_expon_2", parameter_type=ParameterType.FLOAT, lower=0.001, upper=0.1),
RangeParameter(name="lambda_expon_3", parameter_type=ParameterType.FLOAT, lower=0.001, upper=0.1),
RangeParameter(name="lambda_expon_4", parameter_type=ParameterType.FLOAT, lower=0.001, upper=0.1),
]
)
self.exp = SimpleExperiment(
name="0",
search_space=search_space,
evaluation_function=self.evaluate_parameter,
objective_name="adj_return"
)
def initialization_trials(self):
sobol = Models.SOBOL(self.exp.search_space)
for i in range(C.INITIALIZATION_TRIALS):
self.exp.new_trial(generator_run=sobol.gen(1))
def optimization_trials(self):
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
device = torch.device("cpu")
print(device)
for i in range(C.OPTIMIZATION_TRIALS):
print('Running GP+EI optimization trial', i+1)
# Reinitialize GP+EI model at each step with updated data.
#gpei = Models.GPEI(experiment=self.exp, data=self.exp.eval(), dtype=torch.float32, device=device)
gpei = Models.GPEI(experiment=self.exp, data=self.exp.eval())
batch = self.exp.new_trial(generator_run=gpei.gen(1))
def get_best_parameters(self):
_, trial = list(self.exp.trials.items())[-1]
print(trial)
trial.generator_run
gr = trial.generator_run
best_arm, best_arm_predictions = gr.best_arm_predictions
best_parameters = best_arm.parameters
self.best_parameters = best_parameters
print('the best parameters are',best_parameters)
def store_parameters(self):
cwd = os.getcwd()
#cwd = os.path.join(cwd, 'data_folder')
parameter_file = 'best_parameter_from_multi_armed_bandit.json'
cwd = os.path.join(cwd,parameter_file)
with open(cwd, 'w') as statusFile:
statusFile.write(jsonpickle.encode(self.best_parameters))
cwd = os.getcwd()
parameter_file = 'best_parameter_from_direct_experiment.json'
cwd = os.path.join(cwd,parameter_file)
with open(cwd, 'w') as statusFile:
statusFile.write(jsonpickle.encode(self.best_parameters_from_experiment))
def run_optimization(self):
self.init_search_space()
print('Begin initialization trials')
self.initialization_trials()
self.optimization_trials()
self.get_best_parameters()
self.store_parameters()
def mems_exp(cfg):
log.info("============= Configuration =============")
log.info(f"Config:\n{cfg.pretty()}")
log.info("=========================================")
search_space = gen_search_space(cfg.problem)
outcome_con = gen_outcome_constraints(cfg.problem)
exp = SimpleExperiment(
name=cfg.problem.name,
search_space=SearchSpace(search_space),
evaluation_function=jumper,
objective_name="Energy_(uJ)",
minimize=False,
outcome_constraints=outcome_con,
)
optimization_config = OptimizationConfig(objective=Objective(
metric=MEMsMetric(name="Energy_(uJ)"),
minimize=False,
), )
class MyRunner(Runner):
def run(self, trial):
return {"name": str(trial.index)}
exp.runner = MyRunner()
exp.optimization_config = optimization_config
from ax.utils.notebook.plotting import render, init_notebook_plotting
from ax.plot.contour import plot_contour
print(f"Running {cfg.bo.random} Sobol initialization trials...")
sobol = Models.SOBOL(exp.search_space)
num_search = cfg.bo.random
for i in range(num_search):
exp.new_trial(generator_run=sobol.gen(1))
exp.trials[len(exp.trials) - 1].run()
# data = exp.fetch_data()
num_opt = cfg.bo.optimized
for i in range(num_opt):
if (i % 5) == 0 and cfg.plot_during:
plot = plot_contour(
model=gpei,
param_x="N",
param_y="L",
metric_name="Energy_(uJ)",
)
data = plot[0]['data']
lay = plot[0]['layout']
render(plot)
print(f"Running GP+EI optimization trial {i + 1}/{num_opt}...")
# Reinitialize GP+EI model at each step with updated data.
batch = exp.new_trial(generator_run=gpei.gen(1))
gpei = Models.BOTORCH(experiment=exp, data=exp.eval())
plot_learn = plot_learning(exp, cfg)
# go.Figure(plot_learn).show()
save_fig([plot_learn], "optimize")
from ax.utils.notebook.plotting import render, init_notebook_plotting
from ax.plot.contour import plot_contour
plot = plot_contour(model=gpei,
param_x="N",
param_y="L",
metric_name="Energy_(uJ)",
lower_is_better=cfg.metric.minimize)
save_fig(plot, dir=f"N_L_Energy")
# render(plot)
plot = plot_contour(model=gpei,
param_x="N",
param_y="w",
metric_name="Energy_(uJ)",
lower_is_better=cfg.metric.minimize)
# render(plot)
save_fig(plot, dir=f"N_w_Energy")
plot = plot_contour(model=gpei,
param_x="w",
param_y="L",
metric_name="Energy_(uJ)",
lower_is_better=cfg.metric.minimize)
save_fig(plot, dir=f"w_L_Energy")
def mems_exp(cfg):
log.info("============= Configuration =============")
log.info(f"Config:\n{cfg.pretty()}")
log.info("=========================================")
raise NotImplementedError("TODO load experimental data and verify likelihood of differnet points")
search_space = gen_search_space(cfg.problem)
outcome_con = gen_outcome_constraints(cfg.problem)
exp = SimpleExperiment(
name=cfg.problem.name,
search_space=SearchSpace(search_space),
evaluation_function=jumper,
objective_name="Energy_(uJ)",
minimize=False,
outcome_constraints=outcome_con,
)
optimization_config = OptimizationConfig(
objective=Objective(
metric=MEMsMetric(name="Energy_(uJ)"),
minimize=False,
),
)
class MyRunner(Runner):
def run(self, trial):
return {"name": str(trial.index)}
exp.runner = MyRunner()
exp.optimization_config = optimization_config
from ax.utils.notebook.plotting import render, init_notebook_plotting
from ax.plot.contour import plot_contour
print(f"Running {cfg.bo.random} Sobol initialization trials...")
sobol = Models.SOBOL(exp.search_space)
num_search = cfg.bo.random
for i in range(num_search):
exp.new_trial(generator_run=sobol.gen(1))
exp.trials[len(exp.trials) - 1].run()
# data = exp.fetch_data()
num_opt = cfg.bo.optimized
for i in range(num_opt):
if (i % 5) == 0 and cfg.plot_during:
plot = plot_contour(model=gpei,
param_x="N",
param_y="L",
metric_name="Energy_(uJ)", )
data = plot[0]['data']
lay = plot[0]['layout']
render(plot)
print(f"Running GP+EI optimization trial {i + 1}/{num_opt}...")
# Reinitialize GP+EI model at each step with updated data.
batch = exp.new_trial(generator_run=gpei.gen(1))
gpei = Models.BOTORCH(experiment=exp, data=exp.eval())
gpei = Models.BOTORCH(experiment=exp, data=exp.eval())
from ax.models.torch.botorch_defaults import predict_from_model
import torch
X = torch.Tensor([[2, 7e-4, 1e-4], [1, 5e-4, 1e-4]]).double()
mean, cov = predict_from_model(gpei.model.model, X)
# X(Tensor) – n x d parameters
ll = log_likelihood(X, mean, cov)
plot_ll(ll)
# We need to define a search space for our experiment that defines the parameters and the set of feasible values.
search_space = SearchSpace(parameters=[
RangeParameter(
name="x1", parameter_type=ParameterType.FLOAT, lower=-5, upper=10),
RangeParameter(
name="x2", parameter_type=ParameterType.FLOAT, lower=0, upper=15),
])
# Third, we make a `SimpleExperiment` — note that the `objective_name` needs to be one of the metric names returned by the evaluation function.
exp = SimpleExperiment(
name="test_branin",
search_space=search_space,
evaluation_function=branin,
objective_name=
"branin", # This name has to coincide with the name of the function to call
minimize=True,
)
# We use the Sobol generator to create 5 (quasi-) random initial point in the search space. Calling `batch_trial` will cause Ax to evaluate the underlying `branin` function at the generated points, and automatically keep track of the results.
sobol = get_sobol(exp.search_space)
exp.new_batch_trial(generator_run=sobol.gen(5))
# To run our custom botorch model inside the Ax optimization loop, we can use the `get_botorch` factory function from `ax.modelbridge.factory`. Any keyword arguments given to this function are passed through to the `BotorchModel` constructor. To use our custom model, we just need to pass our newly minted `_get_and_fit_simple_custom_gp` function to `get_botorch` using the `model_constructor` argument.
#
# **Note:** `get_botorch` by default automatically applies a number of parameter transformations (e.g. to normalize input data or standardize output data). This is typically what you want for standard use cases with continuous parameters. If your model expects raw parameters, make sure to pass in `transforms=[]` to avoid any transformations to take place. See the [Ax documentation](https://ax.dev/docs/models.html#transforms) for additional information on how transformations in Ax work.
# #### Run the optimization loop
#
class optimization():
def __init__(self):
self.max_worker = multiprocessing.cpu_count()
if self.max_worker > 1:
self.max_worker -= 5
#self.max_worker = 1
self.best_obj_so_far = float('-inf')
self.exp = None #define an experiment for ax
self.best_parameters_from_experiment = None
self.global_iteration_count = 0
def evaluate_parameter(self,parameter, weight=None):
total_returns = []
q = multiprocessing.Queue(maxsize = self.max_worker)
for iteration_index in range(0,C.ROUNDS_OF_WORKER):
p_list = []
for worker in range(0,self.max_worker):
try:
p = multiprocessing.Process(target = self.get_one_sample_score,\
args = (q,parameter))
p.start()
p_list.append(p)
except:
pass
for j in range(len(p_list)):
res = q.get()
total_returns.append(res[0])
self.global_iteration_count += 1
if np.mean(total_returns)>self.best_obj_so_far:
self.best_obj_so_far = np.mean(total_returns)
self.best_parameters_from_experiment = parameter
print('the best objective so far is',self.best_obj_so_far)
print('the associated best parameters are',parameter)
print('the current time is',datetime.datetime.now())
cwd = os.getcwd()
parameter_file = 'best_parameter_from_direct_experiment.json'
cwd = os.path.join(cwd,parameter_file)
with open(cwd, 'w') as statusFile:
statusFile.write(jsonpickle.encode(self.best_parameters_from_experiment))
print('this is global iteration',self.global_iteration_count)
print('the objective for ths iteration is',np.mean(total_returns))
print('the current time is',datetime.datetime.now())
print(' ')
optimization_objective = (np.mean(total_returns) - np.std(total_returns) * 0.5, np.std(total_returns))
return {'adj_return': optimization_objective}
def get_one_sample_score(self,q,free_parameters):
#the size of each gaussian copula cannot be too large
#otherwise, the CDF will almost always be 0.
#the max dimension of the copula is 3
#so, we will use a hierarchical structure of gaussian copulas
#with each of having dimension of less than 3.
cov_input = [free_parameters['cov_para_1'],free_parameters['cov_para_2'],free_parameters['cov_para_3']]
marginal_input = [free_parameters['beta_1a'],free_parameters['beta_1b'],
free_parameters['beta_2a'],free_parameters['beta_2b'],
free_parameters['beta_3a'],free_parameters['beta_3b']]
dist_1 = utils.construct_a_copula(cov_input,marginal_input)
cov_input = [free_parameters['cov_para_4']]
marginal_input = [free_parameters['beta_4a'],free_parameters['beta_4b'],
free_parameters['beta_5a'],free_parameters['beta_5b']]
dist_2 = utils.construct_a_copula(cov_input,marginal_input)
#other parameters needed for transforming the features
lambda_expon_1 = free_parameters['lambda_expon_1']
lambda_expon_2 = free_parameters['lambda_expon_2']
lambda_expon_3 = free_parameters['lambda_expon_3']
lambda_expons = [lambda_expon_1,lambda_expon_2,lambda_expon_3]
#now, we begin to simulate trading
#first, initialize the observation
locol_env = trading_vix_objective_2.trading_vix()
this_trajectory_reward = []
current_feature = locol_env.reset()
for time_index in range(0,200):
#compute an action given current observation
transformed_features = []
for feature_index in range(len(lambda_expons)):
transformation = expon.cdf(current_feature[feature_index,0],scale = 1.0/lambda_expons[feature_index])
transformed_features.append(transformation)
transformed_features = np.asarray(transformed_features)
transformed_features = np.reshape(transformed_features,(1,-1))
data_point_for_copula_1 = pd.DataFrame(transformed_features)
copula_1_output = dist_1.cdf(data_point_for_copula_1)
data_point_for_copula_2 = np.asarray([copula_1_output,current_feature[-1,:][0]])
data_point_for_copula_2 = np.reshape(data_point_for_copula_2,(1,-1))
data_point_for_copula_2 = pd.DataFrame(data_point_for_copula_2)
copula_2_output = dist_2.cdf(data_point_for_copula_2)
#print('action is',copula_1_output)
#apply the action to the environment
current_feature, reward = locol_env.step(copula_1_output)
#record reward
this_trajectory_reward.append(reward)
#final time step,get the long term reward
reward = locol_env.final()
this_trajectory_reward.append(reward)
#print('the sum of the reward is',np.sum(this_trajectory_reward))
objective = np.sum(this_trajectory_reward)
if objective == 0:
objective = -1e9
q.put([objective])
def init_search_space(self):
search_space = SearchSpace(
parameters=[
RangeParameter(name="cov_para_1", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
RangeParameter(name="cov_para_2", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
RangeParameter(name="cov_para_3", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
RangeParameter(name="cov_para_4", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
# RangeParameter(name="cov_para_5", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
# RangeParameter(name="cov_para_6", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
RangeParameter(name="beta_1a", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_1b", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_2a", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_2b", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_3a", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_3b", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_4a", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_4b", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_5a", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_5b", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="lambda_expon_1", parameter_type=ParameterType.FLOAT, lower=0.001, upper=0.1),
RangeParameter(name="lambda_expon_2", parameter_type=ParameterType.FLOAT, lower=0.001, upper=0.1),
RangeParameter(name="lambda_expon_3", parameter_type=ParameterType.FLOAT, lower=0.001, upper=0.1),
]
)
self.exp = SimpleExperiment(
name="0",
search_space=search_space,
evaluation_function=self.evaluate_parameter,
objective_name="adj_return"
)
def initialization_trials(self):
sobol = Models.SOBOL(self.exp.search_space)
for i in range(C.INITIALIZATION_TRIALS):
self.exp.new_trial(generator_run=sobol.gen(1))
def optimization_trials(self):
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
device = torch.device("cpu")
print(device)
for i in range(C.OPTIMIZATION_TRIALS):
print('Running GP+EI optimization trial', i+1)
# Reinitialize GP+EI model at each step with updated data.
#gpei = Models.GPEI(experiment=self.exp, data=self.exp.eval(), dtype=torch.float32, device=device)
gpei = Models.GPEI(experiment=self.exp, data=self.exp.eval())
batch = self.exp.new_trial(generator_run=gpei.gen(1))
def get_best_parameters(self):
_, trial = list(self.exp.trials.items())[-1]
print(trial)
trial.generator_run
gr = trial.generator_run
best_arm, best_arm_predictions = gr.best_arm_predictions
best_parameters = best_arm.parameters
self.best_parameters = best_parameters
print('the best parameters are',best_parameters)
def store_parameters(self):
cwd = os.getcwd()
#cwd = os.path.join(cwd, 'data_folder')
parameter_file = 'index_copula_parameter.json'
cwd = os.path.join(cwd,parameter_file)
with open(cwd, 'w') as statusFile:
statusFile.write(jsonpickle.encode(self.best_parameters))
cwd = os.getcwd()
parameter_file = 'best_parameter_from_direct_experiment.json'
cwd = os.path.join(cwd,parameter_file)
with open(cwd, 'w') as statusFile:
statusFile.write(jsonpickle.encode(self.best_parameters_from_experiment))
def run_optimization(self):
self.init_search_space()
print('Begin initialization trials')
self.initialization_trials()
self.optimization_trials()
self.get_best_parameters()
self.store_parameters()
# mll = ExactMarginalLogLikelihood(gp.likelihood, gp)
# fit_gpytorch_model(mll);
# return gp
search_space = SearchSpace(parameters=[
RangeParameter(
name="x1", parameter_type=ParameterType.FLOAT, lower=-5, upper=10),
RangeParameter(
name="x2", parameter_type=ParameterType.FLOAT, lower=0, upper=15),
])
exp = SimpleExperiment(
name="test_branin",
search_space=search_space,
evaluation_function=branin,
objective_name=
"branin", # This name has to coincide with the name of the function to call
minimize=True,
)
sobol = get_sobol(exp.search_space)
exp.new_batch_trial(generator_run=sobol.gen(5))
for i in range(5):
print(f"Running optimization batch {i+1}/5...")
# user added:
# get_botorch is a self-contained macro-framework that does everything for you:
# 1) It fits the GP model to the new data
# 2) If optimizes the acquisition function and provides the next candidate
# 3) It evaluates such candidate (next point)
mdl = Model(data_mat, lags, n_oos, n_val, prediction_range,
target_vars_inds, params)
search_space = SearchSpace(parameters=[
RangeParameter(name="lr", lower=1.0e-5, upper=1.0e-1,
parameter_type=ParameterType.FLOAT),
RangeParameter(name="lr_change", lower=0.5, upper=1.0,
parameter_type=ParameterType.FLOAT),
RangeParameter(name="leafes", lower=2, upper=1000,
parameter_type=ParameterType.INT)]
)
experiment = SimpleExperiment(
name = f"weather_lbgm_{dt.datetime.today().strftime('%d-%m-%Y')}",
search_space = search_space,
evaluation_function = mdl.obj_fun,
)
sobol = Models.SOBOL(experiment.search_space)
for i in range(n_random_trials):
experiment.new_trial(generator_run=sobol.gen(1))
best_arm = None
for i in range(n_searches):
gpei = Models.GPEI(experiment=experiment, data=experiment.eval())
generator_run = gpei.gen(1)
best_arm, _ = generator_run.best_arm_predictions
experiment.new_trial(generator_run=generator_run)
best_para_ax = best_arm.parameters
def pid(cfg):
env_name = cfg.env.params.name
env = gym.make(env_name)
env.reset()
full_rewards = []
exp_cfg = cfg.experiment
from learn.utils.plotly import hv_characterization
hv_characterization()
def compare_control(env, cfg, save=True):
import torch
from learn.control.pid import PidPolicy
controllers = []
labels = []
metrics = []
# PID baselines
# /Users/nato/Documents/Berkeley/Research/Codebases/dynamics-learn/sweeps/2020-04-14/11-12-02
# from learn.simulate_sac import *
# Rotation policy
sac_policy1 = torch.load(
'/Users/nato/Documents/Berkeley/Research/Codebases/dynamics-learn/outputs/2020-03-24/18-32-26/trial_70000.dat'
)
controllers.append(sac_policy1['policy'])
labels.append("SAC - Rotation")
metrics.append(0)
# Living reward policy
sac_policy2 = torch.load(
'/Users/nato/Documents/Berkeley/Research/Codebases/dynamics-learn/outputs/2020-03-24/18-31-45/trial_35000.dat'
)
controllers.append(sac_policy2['policy'])
labels.append("SAC - Living")
metrics.append(1)
# Square cost policy
# sac_policy2 = torch.load(
# '/Users/nato/Documents/Berkeley/Research/Codebases/dynamics-learn/sweeps/2020-03-25/20-30-47/metric.name=Square,robot=iono_sim/26/trial_40000.dat')
controllers.append(sac_policy2['policy'])
labels.append("SAC - Square")
metrics.append(2)
# un-Optimized PID parameters
pid_params = [[2531.917, 61.358, 33.762], [2531.917, 61.358, 33.762]]
pid = PidPolicy(cfg)
pid.set_params(pid_params)
controllers.append(pid)
labels.append("PID - temp")
metrics.append(0)
controllers.append(pid)
labels.append("PID - temp")
metrics.append(1)
# Optimized PID parameters
pid_params = [[2531.917, 61.358, 3333.762],
[2531.917, 61.358, 3333.762]]
pid = PidPolicy(cfg)
pid.set_params(pid_params)
controllers.append(pid)
labels.append("PID - improved")
metrics.append(2)
from learn.control.mpc import MPController
cfg.policy.mode = 'mpc'
# dynam_model = torch.load(
# '/Users/nato/Documents/Berkeley/Research/Codebases/dynamics-learn/outputs/2020-03-25/10-45-17/trial_1.dat')
dynam_model = torch.load(
'/Users/nato/Documents/Berkeley/Research/Codebases/dynamics-learn/sweeps/2020-03-25/20-30-57/metric.name=Rotation,robot=iono_sim/14/trial_9.dat'
)
mpc = MPController(env, dynam_model['model'], cfg)
controllers.append(mpc)
labels.append("MPC - 1")
metrics.append(0)
controllers.append(mpc)
labels.append("MPC - 2")
metrics.append(1)
controllers.append(mpc)
labels.append("MPC - 3")
metrics.append(2)
import plotly.graph_objects as go
import plotly
colors = [
'#1f77b4', # muted blue
'#ff7f0e', # safety orange
'#2ca02c', # cooked asparagus green
'#d62728', # brick red
'#9467bd', # muted purple
'#8c564b', # chestnut brown
'#e377c2', # raspberry yogurt pink
'#7f7f7f', # middle gray
'#bcbd22', # curry yellow-green
'#17becf' # blue-teal
]
markers = [
"cross",
"circle-open-dot",
"x-open-dot",
"triangle-up-open-dot",
"y-down-open",
"diamond-open-dot",
"hourglass",
"hash",
"star",
"square",
]
m1 = living_reward
m2 = rotation_mat
m3 = squ_cost
eval_metrics = [m1, m2, m3]
metric_names = ["Living", "Rotation", "Square"]
fig = plotly.subplots.make_subplots(
rows=3,
cols=2,
# subplot_titles=["Living", "Rotation", "Square"],
subplot_titles=[
"Pitch",
"Roll",
" ",
" ",
" ",
" ",
],
vertical_spacing=0.03,
horizontal_spacing=0.03,
shared_xaxes=True,
) # go.Figure()
fig_mpc = go.Figure()
fig_sac = go.Figure()
pry = [1, 0, 2]
# state0 = 2*env.reset()
# state0 = env.reset()
state0 = np.array([0, np.deg2rad(15), 0, 0, 0, 0])
for i, (con, lab, m) in enumerate(zip(controllers, labels, metrics)):
print(f"Evaluating controller type {lab}")
_ = env.reset()
env.set_state(np.concatenate((np.zeros(6), state0)))
state = state0
states = []
actions = []
rews = []
done = False
# for t in range(cfg.experiment.r_len + 1):
for t in range(500):
if done:
break
if "SAC" in lab:
with torch.no_grad():
with eval_mode(con):
action = con.select_action(state)
if i < 2:
action = np.array([65535, 65535, 65535, 65535
]) * (action + 1) / 2
else:
action = np.array([3000, 3000, 3000, 3000
]) * (action + 1) / 2
else:
action = con.get_action(state, metric=eval_metrics[m])
states.append(state)
actions.append(action)
state, rew, done, _ = env.step(action)
done = done
states = np.stack(states)
actions = np.stack(actions)
pitch = np.degrees(states[:, pry[0]])
roll = np.degrees(states[:, pry[1]])
# deal with markers
num_mark = np.zeros(len(pitch))
mark_every = 50
m_size = 32
start = np.random.randint(0, int(len(pitch) / 10))
num_mark[start::mark_every] = m_size
if "SAC" in lab:
fig_sac.add_trace(
go.Scatter(
y=pitch,
name=metric_names[m], # legendgroup=lab[:3],
# showlegend=(True if (i % 3 == 0) else False),
line=dict(color=colors[m], width=4),
cliponaxis=False,
mode='lines+markers',
marker=dict(color=colors[m],
symbol=markers[-m],
size=num_mark.tolist())))
elif "MPC" in lab:
fig_mpc.add_trace(
go.Scatter(
y=pitch,
name=metric_names[m], # legendgroup=lab[:3],
# showlegend=(True if (i % 3 == 0) else False),
line=dict(color=colors[m], width=4),
cliponaxis=False,
mode='lines+markers',
marker=dict(color=colors[m],
symbol=markers[-m],
size=num_mark.tolist())))
fig.add_trace(
go.Scatter(
y=pitch,
name=lab[:3] + str(m),
legendgroup=lab[:3],
showlegend=(True if (i % 3 == 0) else False),
line=dict(color=colors[int(i / 3)],
width=2), # mode='lines+markers',
# marker=dict(color=colors[i], symbol=markers[i], size=16)
),
row=m + 1,
col=1)
fig.add_trace(
go.Scatter(
y=roll,
name=lab[:3] + str(m),
legendgroup=lab[:3],
showlegend=(False),
line=dict(color=colors[int(i / 3)],
width=2), # mode='lines+markers',
# marker=dict(color=colors[i], symbol=markers[i], size=16)
),
row=m + 1,
col=2)
fig.update_layout(
title='Comparison of Controllers and Reward Functions',
font=dict(family="Times New Roman, Times, serif",
size=24,
color="black"),
legend_orientation="h",
legend=dict(
x=.6,
y=0.07,
bgcolor='rgba(205, 223, 212, .4)',
bordercolor="Black",
),
# xaxis_title='Timestep',
# yaxis_title='Angle (Degrees)',
plot_bgcolor='white',
width=1600,
height=1000,
# xaxis=dict(
# showline=True,
# showgrid=False,
# showticklabels=True, ),
# yaxis=dict(
# showline=True,
# showgrid=False,
# showticklabels=True, ),
)
fig_sac.update_layout( # title='Comparison of SAC Policies',
font=dict(family="Times New Roman, Times, serif",
size=32,
color="black"),
legend_orientation="h",
legend=dict(
x=.35,
y=0.1,
bgcolor='rgba(205, 223, 212, .4)',
bordercolor="Black",
),
# xaxis_title='Timestep',
# yaxis_title='Angle (Degrees)',
showlegend=False,
plot_bgcolor='white',
width=1600,
height=800,
margin=dict(t=5, r=5),
)
fig_mpc.update_layout( # title='Comparison of MPC Policies',
font=dict(family="Times New Roman, Times, serif",
size=32,
color="black"),
legend_orientation="h",
showlegend=False,
legend=dict(
x=.35,
y=0.1,
bgcolor='rgba(205, 223, 212, .4)',
bordercolor="Black",
# ncol= 2,
),
# xaxis_title='Timestep',
# yaxis_title='Angle (Degrees)',
plot_bgcolor='white',
width=1600,
height=800,
margin=dict(t=5, r=5),
)
reg_color = 'rgba(255,60,60,.15)'
fig_sac.add_trace(
go.Scatter(x=[0, 500],
y=[5, 5],
name='Living Region',
legendgroup='Living Region',
fill='tozeroy',
mode='lines',
fillcolor=reg_color,
line=dict(width=0.0,
color=reg_color))) # fill down to xaxis
fig_sac.add_trace(
go.Scatter(x=[0, 500],
y=[-5, -5],
showlegend=False,
legendgroup='Living Region',
fill='tozeroy',
mode='lines',
fillcolor=reg_color,
line=dict(width=0.0,
color=reg_color))) # fill down to xaxis
fig_mpc.add_trace(
go.Scatter(x=[0, 500],
y=[5, 5],
name='Living Region',
legendgroup='Living Region',
fill='tozeroy',
mode='lines',
fillcolor=reg_color,
line=dict(width=0.0,
color=reg_color))) # fill down to xaxis
fig_mpc.add_trace(
go.Scatter(x=[0, 500],
y=[-5, -5],
showlegend=False,
legendgroup='Living Region',
fill='tozeroy',
mode='lines',
fillcolor=reg_color,
line=dict(width=0.0,
color=reg_color))) # fill down to xaxis
# SOLO
rang_ind = [-20, 20]
fig_sac.update_xaxes(
title_text="Timestep",
range=[0, 500],
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig_sac.update_yaxes(
title_text="Pitch (degrees)",
range=rang_ind,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig_sac.show()
fig_sac.write_image(os.getcwd() + "/compare_sac.pdf")
fig_mpc.update_xaxes(
title_text="Timestep",
range=[0, 500],
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig_mpc.update_yaxes(
title_text="Pitch (degrees)",
range=rang_ind,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig_mpc.show()
fig_mpc.write_image(os.getcwd() + "/compare_mpc.pdf")
# COMPARISON
fig.update_xaxes(
title_text="Timestep",
row=3,
col=1,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_xaxes(
row=2,
col=1,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_xaxes(
row=1,
col=1,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_xaxes(
title_text="Timestep",
row=3,
col=2,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_xaxes(
row=2,
col=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_xaxes(
row=1,
col=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
# fig.update_xaxes(title_text="xaxis 1 title", row=1, col=1)
# fig.update_yaxes(title_text="Roll (Degrees)", row=1, col=1)
rang = [-30, 30]
nticks = 6
fig.update_yaxes(
title_text="Living Rew.",
range=rang,
row=1,
col=1,
nticks=nticks,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_yaxes(
title_text="Rotation Rew.",
range=rang,
row=2,
col=1,
nticks=nticks,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_yaxes(
title_text="Square Cost",
range=rang,
row=3,
col=1,
nticks=nticks,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_yaxes(
range=rang,
row=1,
col=2,
nticks=nticks,
showticklabels=False,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_yaxes(
range=rang,
row=2,
col=2,
nticks=nticks,
showticklabels=False,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_yaxes(
range=rang,
row=3,
col=2,
nticks=nticks,
showticklabels=False,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
print(f"Plotting {len(labels)} control responses")
# save = False
# if save:
# fig.write_image(os.getcwd() + "compare.png")
# else:
# fig.show()
#
# return fig
# compare_control(env, cfg, save=True)
# quit()
plot_results(logx=False, save=True, mpc=False)
quit()
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # Evalutation Function # # # # # # # # # # # # # # # # # # # #
def bo_rollout_wrapper(params, weights=None): # env, controller, exp_cfg):
pid_1 = [params["pitch-p"], params["pitch-i"], params["pitch-d"]]
# pid_1 = [params["roll-p"], params["roll-i"],
# params["roll-d"]] # [params["pitch-p"], params["pitch-i"], params["pitch-d"]]
pid_2 = [params["roll-p"], params["roll-i"], params["roll-d"]]
print(
f"Optimizing Parameters {np.round(pid_1, 3)},{np.round(pid_2, 3)}")
pid_params = [[pid_1[0], pid_1[1], pid_1[2]],
[pid_2[0], pid_2[1], pid_2[2]]]
# pid_params = [[1000, 0, 0], [1000, 0, 0]]
pid = PidPolicy(cfg)
pid.set_params(pid_params)
cum_cost = []
r = 0
fncs = [squ_cost, living_reward, rotation_mat]
mult_rewards = [[] for _ in range(len(fncs))]
while r < cfg.experiment.repeat:
pid.reset()
states, actions, rews, sim_error = rollout(env, pid, exp_cfg)
# plot_rollout(states, actions, pry=[1, 0, 2])
rewards_full = get_rewards(states, actions, fncs=fncs)
for i, vec in enumerate(rewards_full):
mult_rewards[i].append(vec)
# if sim_error:
# print("Repeating strange simulation")
# continue
# if len(rews) < 400:
# cum_cost.append(-(cfg.experiment.r_len - len(rews)) / cfg.experiment.r_len)
# else:
rollout_cost = np.sum(rews) / cfg.experiment.r_len # / len(rews)
# if rollout_cost > max_cost:
# max_cost = rollout_cost
# rollout_cost += get_reward_euler(states[-1], actions[-1])
cum_cost.append(rollout_cost)
r += 1
std = np.std(cum_cost)
cum_cost = np.mean(cum_cost)
# print(f"Cum. Cost {cum_cost / max_cost}")
# print(f"- Mean Cum. Cost / Rew: {cum_cost}, std dev: {std}")
eval = {
"Square": (np.mean(rewards_full[0]), np.std(rewards_full[0])),
"Living": (np.mean(rewards_full[1]), np.std(rewards_full[1])),
"Rotation": (np.mean(rewards_full[2]), np.std(rewards_full[2]))
}
for n, (key, value) in enumerate(eval.items()):
if n == 0:
print(f"- Square {np.round(value, 4)}")
elif n == 1:
print(f"- Living {np.round(value, 4)}")
else:
print(f"- Rotn {np.round(value, 4)}")
return eval
# return cum_cost.reshape(1, 1), std
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
from ax import (
ComparisonOp,
ParameterType,
RangeParameter,
SearchSpace,
SimpleExperiment,
OutcomeConstraint,
)
exp = SimpleExperiment(
name="PID Control Robot",
search_space=SearchSpace([
RangeParameter(
name=f"roll-p",
parameter_type=ParameterType.FLOAT,
lower=1.0,
upper=10000.0,
log_scale=True,
),
# FixedParameter(name="roll-i", value=0.0, parameter_type=ParameterType.FLOAT),
RangeParameter(
name=f"roll-i",
parameter_type=ParameterType.FLOAT,
lower=0,
upper=1000.0,
log_scale=False,
),
RangeParameter(
name=f"roll-d",
parameter_type=ParameterType.FLOAT,
lower=.1,
upper=5000.0,
log_scale=True,
),
RangeParameter(
name=f"pitch-p",
parameter_type=ParameterType.FLOAT,
lower=1.0,
upper=10000.0,
log_scale=True,
),
RangeParameter(
name=f"pitch-d",
parameter_type=ParameterType.FLOAT,
lower=0,
upper=1000.0,
log_scale=False,
),
RangeParameter(
name=f"pitch-i",
parameter_type=ParameterType.FLOAT,
lower=.1,
upper=5000.0,
log_scale=True,
),
# FixedParameter(name="pitch-i", value=0.0, parameter_type=ParameterType.FLOAT),
]),
evaluation_function=bo_rollout_wrapper,
objective_name=cfg.metric.name,
minimize=cfg.metric.minimize,
outcome_constraints=[],
)
from ax.storage.metric_registry import register_metric
from ax.storage.runner_registry import register_runner
class GenericMetric(Metric):
def fetch_trial_data(self, trial):
records = []
for arm_name, arm in trial.arms_by_name.items():
params = arm.parameters
mean, sem = bo_rollout_wrapper(params)
records.append({
"arm_name": arm_name,
"metric_name": self.name,
"mean": mean,
"sem": sem,
"trial_index": trial.index,
})
return Data(df=pd.DataFrame.from_records(records))
class MyRunner(Runner):
def run(self, trial):
return {"name": str(trial.index)}
optimization_config = OptimizationConfig(objective=Objective(
metric=GenericMetric(name=cfg.metric.name),
minimize=cfg.metric.minimize,
), )
register_metric(GenericMetric)
register_runner(MyRunner)
exp.runner = MyRunner()
exp.optimization_config = optimization_config
log.info(f"Running experiment, metric name {cfg.metric.name}")
log.info(f"Running Sobol initialization trials...")
sobol = Models.SOBOL(exp.search_space)
num_search = cfg.bo.random
for i in range(num_search):
exp.new_trial(generator_run=sobol.gen(1))
exp.trials[len(exp.trials) - 1].run()
import plotly.graph_objects as go
gpei = Models.BOTORCH(experiment=exp, data=exp.eval())
objectives = ["Living", "Square", "Rotation"]
def plot_all(model, objectives, name="", rend=False):
for o in objectives:
plot = plot_contour(
model=model,
param_x="roll-p",
param_y="roll-d",
metric_name=o,
)
plot[0]['layout']['title'] = o
data = plot[0]['data']
lay = plot[0]['layout']
for i, d in enumerate(data):
if i > 1:
d['cliponaxis'] = False
fig = {
"data": data,
"layout": lay,
}
go.Figure(fig).write_image(name + o + ".png")
if rend: render(plot)
plot_all(gpei, objectives, name="Random fit-")
num_opt = cfg.bo.optimized
for i in range(num_opt):
log.info(f"Running GP+EI optimization trial {i + 1}/{num_opt}...")
# Reinitialize GP+EI model at each step with updated data.
batch = exp.new_trial(generator_run=gpei.gen(1))
gpei = Models.BOTORCH(experiment=exp, data=exp.eval())
if ((i + 1) % 10) == 0:
plot_all(gpei,
objectives,
name=f"optimizing {str(i + 1)}-",
rend=False)
from ax.plot.exp_utils import exp_to_df
best_arm, _ = gpei.gen(1).best_arm_predictions
best_parameters = best_arm.parameters
log.info(f"Best parameters {best_parameters}")
experiment_log = {
"Exp": exp_to_df(exp=exp),
"Cfg": cfg,
"Best_param": best_parameters,
}
log.info("Printing Parameters")
log.info(exp_to_df(exp=exp))
save_log(cfg, exp, experiment_log)
fig_learn = plot_learning(exp, cfg)
fig_learn.write_image("learning" + ".png")
fig_learn.show()
plot_all(gpei, objectives, name=f"FINAL -", rend=True)
def run_experiment(function=None,
n_dimensions=None,
noise_std=1e-3,
n_initial_evaluations=None,
evaluation_budget=None,
seed=None,
kernel='rbf',
outputscale=None,
lengthscale=None,
nu=None,
n_mixtures=None,
mixture_means_constraint='positive',
likelihood_type='gaussian',
optimiser_type='sgd',
learning_rate=1e-1,
training_iterations=50,
acquisition_function='nei',
visualise=False,
vis_density=None,
log_dir=Path('./'),
observer=None):
problem, search_space = define_problem(function,
n_dimensions=n_dimensions,
noise_std=noise_std,
observer=observer,
visualise=visualise,
vis_density=vis_density,
log_dir=log_dir)
model_constructor = get_model_constructor(
kernel,
likelihood_type,
optimiser_type,
learning_rate,
training_iterations,
outputscale=outputscale,
lengthscale=lengthscale,
nu=nu,
n_mixtures=n_mixtures,
n_dimensions=len(search_space.parameters),
mixture_means_constraint=mixture_means_constraint)
experiment = SimpleExperiment(name='function_experiment',
search_space=search_space,
evaluation_function=problem,
objective_name='function')
sobol(experiment, n_initial_evaluations, seed)
bayes_opt_loop(experiment,
model_constructor,
acquisition_function,
evaluation_budget,
n_initial_evaluations=n_initial_evaluations,
visualise=visualise,
vis_start=search_space.parameters['x0'].lower,
vis_end=search_space.parameters['x0'].upper,
vis_density=vis_density,
function_name=function,
kernel_name=kernel,
log_dir=log_dir,
observer=observer)
save_cumulative_regrets(observer.current['results'][-1], log_dir=log_dir)
if visualise:
bo_vis.regret(
np.array(observer.current['results'][-1]['regret']['location']),
np.array(observer.current['results'][-1]['regret']['function']),
n_initial_evaluations=n_initial_evaluations,
function_name=function,
log_dir=log_dir,
)
def optimize(self):
SOBOL_TRIALS = 75
gpei_list = [50, 30, 10, 0]
parameters = None
for gpei_trials in gpei_list:
try:
search_space = SearchSpace(parameters=[
ChoiceParameter(name='risk_function',
values=['polynomial', 'exponential'],
parameter_type=ParameterType.STRING),
RangeParameter(name='alpha_poly',
lower=1.0,
upper=5.0,
parameter_type=ParameterType.FLOAT),
RangeParameter(name='alpha_exp',
lower=0.0,
upper=1.0,
parameter_type=ParameterType.FLOAT),
RangeParameter(name='exp_threshold',
lower=1.0,
upper=10.0,
parameter_type=ParameterType.FLOAT)
])
experiment = SimpleExperiment(
name='risk_function_parametrisation',
search_space=search_space,
evaluation_function=self.evaluate_parameterization,
objective_name='par10',
minimize=True)
sobol = Models.SOBOL(experiment.search_space)
for _ in range(SOBOL_TRIALS):
experiment.new_trial(generator_run=sobol.gen(1))
best_arm = None
for _ in range(gpei_trials):
gpei = Models.GPEI(experiment=experiment,
data=experiment.eval())
generator_run = gpei.gen(1)
best_arm, _ = generator_run.best_arm_predictions
experiment.new_trial(generator_run=generator_run)
parameters = best_arm.parameters
break
except:
print('GPEI Optimization failed')
if gpei_trials == 0:
# choose expectation if optimization failed for all gpei_trial values
# exp thresholds are dummy variables
parameters = {
'risk_function': 'polynomial',
'alpha_poly': 1.0,
'alpha_exp': 1.0,
'exp_threshold': 1.0
}
else:
continue
return self.resolve_risk_function(parameters['risk_function'],
parameters['alpha_poly'],
parameters['alpha_exp'],
parameters['exp_threshold'])
name='batch', parameter_type=ParameterType.INT, values=[16, 32, 64]),
FixedParameter(
name="n_epochs", parameter_type=ParameterType.INT, value=n_epochs),
FixedParameter(
name="dset", parameter_type=ParameterType.STRING, value=dset),
])
dset = dset.split('/')[-1]
if type == 'ner':
transformer = TransformerNER()
else:
transformer = TransformerCLS()
# Create Experiment
exp = SimpleExperiment(
name='transformer',
search_space=transformer_search_space,
evaluation_function=transformer.trainer,
objective_name='f1',
)
# Run the optimization and fit a GP on all data
sobol = modelbridge.get_sobol(search_space=exp.search_space)
print(f"\nRunning Sobol initialization trials...\n{'='*40}\n")
for _ in range(init_trials):
exp.new_trial(generator_run=sobol.gen(1))
for i in range(opt_trials):
print(
f"\nRunning GP+EI optimization trial {i+1}/{opt_trials}...\n{'='*40}\n"
)
gpei = modelbridge.get_GPEI(experiment=exp, data=exp.eval())
exp.new_trial(generator_run=gpei.gen(1))
