def tune_fcn(
n_sweeps,
time_suffix,
dataset_name,
dataset,
use_gpu,
output_dir,
model_seed,
params_seed,
verbose,
skip_sweeps=None,
):
n_head_units = RangeParameter(name="n_head_units", parameter_type=ParameterType.INT, lower=8, upper=10)
n_tail_units = RangeParameter(name="n_tail_units", parameter_type=ParameterType.INT, lower=7, upper=9)
order_constraint = OrderConstraint(
lower_parameter=n_tail_units,
upper_parameter=n_head_units,
)
search_space = SearchSpace(
parameters=[
n_head_units,
n_tail_units,
RangeParameter(name="n_head_layers", parameter_type=ParameterType.INT, lower=1, upper=2),
RangeParameter(name="n_tail_layers", parameter_type=ParameterType.INT, lower=1, upper=4),
ChoiceParameter(name="dropout", parameter_type=ParameterType.FLOAT, values=[0.0, 0.1, 0.2, 0.3]),
RangeParameter(name="learning_rate", parameter_type=ParameterType.FLOAT, lower=1e-4, upper=1e-2, log_scale=True),
],
parameter_constraints=[order_constraint]
)
sobol = get_sobol(search_space=search_space, seed=params_seed)
sweeps = sobol.gen(n=n_sweeps).arms
if skip_sweeps is not None:
sweeps = sweeps[skip_sweeps:]
for i, sweep in enumerate(sweeps):
train_fcn(
experiment_name="%s_%d_%s" % (dataset_name, i, time_suffix),
dataset=dataset,
batch_size=1024,
device="cuda" if use_gpu else "cpu",
report_frequency=100,
epochs=float("inf"),
output_dir=output_dir,
model_seed=model_seed,
verbose=verbose,
**sweep.parameters,
)
def gen_search_space(cfg):
l = []
for key, item in cfg.space.items():
if item.value_type == 'float':
typ = ParameterType.FLOAT
elif item.value_type == 'int':
typ = ParameterType.INT
elif item.value_type == 'bool':
typ == Parameter.BOOL
else:
raise ValueError("invalid search space value type")
if item.type == 'range':
ss = RangeParameter(
name=key,
parameter_type=typ,
lower=item.bounds[0],
upper=item.bounds[1],
log_scale=item.log_scale,
)
elif item.type == 'fixed':
ss = FixedParameter(name=key,
value=item.bounds,
parameter_type=typ)
elif item.type == 'choice':
ss = ChoiceParameter(name=key,
parameter_type=typ,
values=item.bounds)
else:
raise ValueError("invalid search space parameter type")
l.append(ss)
return l
def tune_catboost(
n_sweeps,
time_suffix,
dataset_name,
dataset,
use_gpu,
output_dir,
model_seed,
params_seed,
verbose,
skip_sweeps=None,
):
search_space = SearchSpace(parameters=[
RangeParameter(name="learning_rate", parameter_type=ParameterType.FLOAT, lower=np.exp(-5), upper=1.0, log_scale=True),
RangeParameter(name="l2_leaf_reg", parameter_type=ParameterType.FLOAT, lower=1, upper=10, log_scale=True),
RangeParameter(name="subsample", parameter_type=ParameterType.FLOAT, lower=0, upper=1),
RangeParameter(name="leaf_estimation_iterations", parameter_type=ParameterType.INT, lower=1, upper=10),
RangeParameter(name="random_strength", parameter_type=ParameterType.INT, lower=1, upper=20),
])
sobol = get_sobol(search_space=search_space, seed=params_seed)
sweeps = sobol.gen(n=n_sweeps).arms
if skip_sweeps is not None:
sweeps = sweeps[skip_sweeps:]
for i, sweep in enumerate(sweeps):
train_catboost(
max_trees=2048,
experiment_name="%s_%d_%s" % (dataset_name, i, time_suffix),
dataset=dataset,
device="GPU" if use_gpu else "CPU",
output_dir=output_dir,
model_seed=model_seed,
verbose=verbose,
report_frequency=100,
**sweep.parameters,
)
def __init__(self, serialized_filepath=None):
# Give ourselves the ability to resume this experiment later.
self.serialized_filepath = serialized_filepath
if serialized_filepath is not None and os.path.exists(
serialized_filepath):
with open(serialized_filepath, "r") as f:
serialized = json.load(f)
self.initialize_from_json_snapshot(serialized)
else:
# Create a CoreAxClient.
search_space = SearchSpace(parameters=[
RangeParameter(
"x", ParameterType.FLOAT, lower=12.2, upper=602.2),
])
optimization_config = OptimizationConfig(
objective=MultiObjective(
metrics=[
# Currently MultiObjective doesn't work with
# lower_is_better=True.
# https://github.com/facebook/Ax/issues/289
Metric(name="neg_distance17", lower_is_better=False),
Metric(name="neg_distance33", lower_is_better=False)
],
minimize=False,
), )
generation_strategy = choose_generation_strategy(
search_space,
enforce_sequential_optimization=False,
no_max_parallelism=True,
num_trials=NUM_TRIALS,
num_initialization_trials=NUM_RANDOM)
super().__init__(experiment=Experiment(
search_space=search_space,
optimization_config=optimization_config),
generation_strategy=generation_strategy,
verbose=True)
def tune_tabnet(
n_sweeps,
time_suffix,
dataset_name,
dataset,
use_gpu,
output_dir,
model_seed,
params_seed,
verbose,
skip_sweeps=None,
):
search_space = SearchSpace(
parameters=[
ChoiceParameter(name="n_d", parameter_type=ParameterType.INT, values=[8, 16, 32, 64]),
RangeParameter(name="n_steps", parameter_type=ParameterType.INT, lower=3, upper=10),
RangeParameter(name="gamma", parameter_type=ParameterType.FLOAT, lower=1, upper=2),
RangeParameter(name="n_independent", parameter_type=ParameterType.INT, lower=1, upper=5),
RangeParameter(name="n_shared", parameter_type=ParameterType.INT, lower=1, upper=5),
RangeParameter(name="learning_rate", parameter_type=ParameterType.FLOAT, lower=1e-3, upper=2e-2, log_scale=True),
RangeParameter(name="lambda_sparse", parameter_type=ParameterType.FLOAT, lower=1e-5, upper=1e-3, log_scale=True),
ChoiceParameter(name="mask_type", parameter_type=ParameterType.STRING, values=["sparsemax", "entmax"]),
]
)
sobol = get_sobol(search_space=search_space, seed=params_seed)
sweeps = sobol.gen(n=n_sweeps).arms
if skip_sweeps is not None:
sweeps = sweeps[skip_sweeps:]
for i, sweep in enumerate(sweeps):
train_tabnet(
experiment_name="%s_%d_%s" % (dataset_name, i, time_suffix),
dataset=dataset,
batch_size=1024,
device="cuda" if use_gpu else "cpu",
epochs=15,
patience=5,
output_dir=output_dir,
model_seed=42,
verbose=int(verbose),
**sweep.parameters
)
def init_search_space(self):
search_space = SearchSpace(
parameters=[
RangeParameter(name="cov_para_1", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
RangeParameter(name="cov_para_2", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
RangeParameter(name="cov_para_3", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
RangeParameter(name="cov_para_4", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
RangeParameter(name="cov_para_5", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
RangeParameter(name="cov_para_6", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
RangeParameter(name="cov_para_7", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
RangeParameter(name="cov_para_8", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
RangeParameter(name="cov_para_9", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
RangeParameter(name="cov_para_10", parameter_type=ParameterType.FLOAT, lower=-0.9, upper=0.9),
RangeParameter(name="beta_1a", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_1b", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_2a", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_2b", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_3a", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_3b", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_4a", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_4b", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_5a", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="beta_5b", parameter_type=ParameterType.FLOAT, lower=0, upper=20),
RangeParameter(name="lambda_expon_1", parameter_type=ParameterType.FLOAT, lower=0.001, upper=0.1),
RangeParameter(name="lambda_expon_2", parameter_type=ParameterType.FLOAT, lower=0.001, upper=0.1),
RangeParameter(name="lambda_expon_3", parameter_type=ParameterType.FLOAT, lower=0.001, upper=0.1),
RangeParameter(name="lambda_expon_4", parameter_type=ParameterType.FLOAT, lower=0.001, upper=0.1),
]
)
self.exp = SimpleExperiment(
name="0",
search_space=search_space,
evaluation_function=self.evaluate_parameter,
objective_name="adj_return"
)
def create_params():
ax_params = []
ax_constraints = []
# continuous parameters with fixed bounds
for i in range(19):
param_name = SAE.fmincon.params.at[i, 'variable']
min_val = SAE.fmincon.params.at[i, 'min']
max_val = SAE.fmincon.params.at[i, 'max']
param = RangeParameter(name=param_name,
parameter_type=ParameterType.FLOAT,
lower=min_val,
upper=max_val)
ax_params.append(param)
# discrete choice parameters
# materials
for i in range(5):
param_name = f'mat_{i}'
param = ChoiceParameter(name=param_name,
parameter_type=ParameterType.INT,
values=list(range(13)))
ax_params.append(param)
# rear tire type
param = ChoiceParameter(name='rear_tire',
parameter_type=ParameterType.INT,
values=list(range(7)))
ax_params.append(param)
# front tire type
param = ChoiceParameter(name='front_tire',
parameter_type=ParameterType.INT,
values=list(range(7)))
ax_params.append(param)
# engine type
param = ChoiceParameter(name='engine',
parameter_type=ParameterType.INT,
values=list(range(21)))
ax_params.append(param)
# brake type
param = ChoiceParameter(name='brakes',
parameter_type=ParameterType.INT,
values=list(range(34)))
ax_params.append(param)
# brake type
param = ChoiceParameter(name='suspension',
parameter_type=ParameterType.INT,
values=list(range(5)))
ax_params.append(param)
# continuous params bounded by other continuous params
hrw_min = 0.025
hfw_min = 0.025
hsw_min = 0.025
hc_min = 0.5
lfw_min = 0.05
hia_min = 0.1
# Here, we use maximum ranges. Some of these are narrowed down below
ax_params.append(
RangeParameter(name='yrw',
parameter_type=ParameterType.FLOAT,
lower=.500 + hrw_min / 2,
upper=1.200 - hrw_min / 2))
ax_params.append(
RangeParameter(name='yfw',
parameter_type=ParameterType.FLOAT,
lower=0.03 + hfw_min,
upper=.250 - hfw_min / 2))
ax_params.append(
RangeParameter(name='ysw',
parameter_type=ParameterType.FLOAT,
lower=0.03 + hsw_min / 2,
upper=.250 - hsw_min / 2))
ax_params.append(
RangeParameter(name='yc',
parameter_type=ParameterType.FLOAT,
lower=0.03 + hc_min / 2,
upper=1.2 - hc_min / 2))
ax_params.append(
RangeParameter(name='lia',
parameter_type=ParameterType.FLOAT,
lower=0.2,
upper=.700 - lfw_min))
ax_params.append(
RangeParameter(name='yia',
parameter_type=ParameterType.FLOAT,
lower=0.03 + hia_min / 2,
upper=1.200 - hia_min / 2))
# yrw > .500 + hrw/2
# <=> 0.5*hrw -1.0*yrw < -0.5
ax_constraints.append(
ParameterConstraint(constraint_dict={
'yrw': -1.0,
'hrw': 0.5
},
bound=-0.5))
# yrw < 1.200 - hrw/2
# <=> 0.5*hrw +1.0*yrw < 1.200
ax_constraints.append(
ParameterConstraint(constraint_dict={
'yrw': 1.0,
'hrw': 0.5
},
bound=1.200))
# yfw > 0.03 + hfw
# <=> -1.0*yfw + 1.0*hfw < -0.03
ax_constraints.append(
ParameterConstraint(constraint_dict={
'yfw': -1.0,
'hfw': 1.0
},
bound=-0.03))
# yfw < 0.25 - hfw/2
# <=> 1.0*yfw + 0.5*hfw < 0.25
ax_constraints.append(
ParameterConstraint(constraint_dict={
'yfw': 1.0,
'hfw': 0.5
},
bound=0.25))
# ysw > 0.03 + hsw/2
# <=> -1.0*ysw + 0.5*hsw < -0.03
ax_constraints.append(
ParameterConstraint(constraint_dict={
'ysw': -1.0,
'hsw': 0.5
},
bound=-0.03))
# ysw < .250 - hsw/2
# <=> 1.0*ysw + 0.5*hsw < 0.25
ax_constraints.append(
ParameterConstraint(constraint_dict={
'ysw': 1.0,
'hsw': 0.5
},
bound=0.25))
# yc > 0.03+hc/2
# <=> -1.0*yc + 0.5hc < -0.03
ax_constraints.append(
ParameterConstraint(constraint_dict={
'yc': -1.0,
'hsw': 0.5
},
bound=-0.03))
# yc < 1.2 - hc/2
# <=> 1.0*yc + 0.5*hc < 1.2
ax_constraints.append(
ParameterConstraint(constraint_dict={
'yc': 1.0,
'hc': 0.5
}, bound=1.2))
# lia > 0.2
# <=> -1.0*lia < -0.2
ax_constraints.append(
ParameterConstraint(constraint_dict={'lia': -1.0}, bound=-0.2))
# lia < 0.7 - lfw
# <=> 1.0*lia + 1.0*lfw < 0.7
ax_constraints.append(
ParameterConstraint(constraint_dict={
'lia': 1.0,
'lfw': 1.0
},
bound=0.7))
# yia > 0.03+hia/2
# <=> -1.0*yia + 0.5*hia < -0.03
ax_constraints.append(
ParameterConstraint(constraint_dict={
'yia': -1.0,
'hia': 0.5
},
bound=-0.03))
# yia < 1.2 - hia/2
# <=> 1.0*yia + 0.5*hia < 1.2
ax_constraints.append(
ParameterConstraint(constraint_dict={
'yia': 1.0,
'hia': 0.5
},
bound=1.2))
# Continuous params with variable bounds
# These are not directly supported in Ax. We give the maximum bounds, and take care of the illegal
# values elsewhere
rt_min = SAE.fmincon.tires['radius'].min()
rt_max = SAE.fmincon.tires['radius'].max()
he_min = SAE.fmincon.motors['Height'].min()
ax_params.append(
RangeParameter(name='wrw',
parameter_type=ParameterType.FLOAT,
lower=0.3,
upper=9 - 2 * rt_min))
ax_params.append(
RangeParameter(name='ye',
parameter_type=ParameterType.FLOAT,
lower=0.03 + he_min / 2,
upper=.500 - he_min / 2))
ax_params.append(
RangeParameter(name='yrsp',
parameter_type=ParameterType.FLOAT,
lower=rt_min,
upper=2 * rt_max))
ax_params.append(
RangeParameter(name='yfsp',
parameter_type=ParameterType.FLOAT,
lower=rt_min,
upper=2 * rt_max))
return ax_params, ax_constraints
# For this tutorial, we use the Branin function, a simple synthetic benchmark function in two dimensions. In an actual application, this could be arbitrarily complicated - e.g. this function could run some costly simulation, conduct some A/B tests, or kick off some ML model training job with the given parameters).
def branin(parameterization, *args):
x1, x2 = parameterization["x1"], parameterization["x2"]
y = (x2 - 5.1 / (4 * np.pi**2) * x1**2 + 5 * x1 / np.pi - 6)**2
y += 10 * (1 - 1 / (8 * np.pi)) * np.cos(x1) + 10
# let's add some synthetic observation noise
y += random.normalvariate(0, 0.1)
return {"branin": (y, 0.0)}
# We need to define a search space for our experiment that defines the parameters and the set of feasible values.
search_space = SearchSpace(parameters=[
RangeParameter(
name="x1", parameter_type=ParameterType.FLOAT, lower=-5, upper=10),
RangeParameter(
name="x2", parameter_type=ParameterType.FLOAT, lower=0, upper=15),
])
# Third, we make a `SimpleExperiment` — note that the `objective_name` needs to be one of the metric names returned by the evaluation function.
exp = SimpleExperiment(
name="test_branin",
search_space=search_space,
evaluation_function=branin,
objective_name=
"branin", # This name has to coincide with the name of the function to call
minimize=True,
)
from ax.core import SearchSpace
from ax.core.parameter_constraint import SumConstraint, OrderConstraint
from ax import RangeParameter, ChoiceParameter, ParameterType
# %% constraint parameters and constraints
betas1 = RangeParameter(name="betas1",
parameter_type=ParameterType.FLOAT,
lower=0.5,
upper=0.9999)
betas2 = RangeParameter(name="betas2",
parameter_type=ParameterType.FLOAT,
lower=0.5,
upper=0.9999)
emb_scaler = RangeParameter(name="emb_scaler",
parameter_type=ParameterType.FLOAT,
lower=0.0,
upper=1.0)
pos_scaler = RangeParameter(name="pos_scaler",
parameter_type=ParameterType.FLOAT,
lower=0.0,
upper=1.0)
order_constraint = OrderConstraint(lower_parameter=betas1,
upper_parameter=betas2)
sum_constraint = SumConstraint(parameters=[emb_scaler, pos_scaler],
is_upper_bound=True,
bound=1.0)
parameter_constraints = [order_constraint, sum_constraint]
# %% search space
search_space = SearchSpace(
parameters=[
expected_improvement, n_random_trials)
ind, best_para_my, y = bo.search(n_searches, 2, 25)
# Use Ax Bayesian Optimization
n_random_trials = 5 # initiate Bayesian optimization with 3 random draws
n_searches = 20
mdl = Model(data_mat, lags, n_oos, n_val, prediction_range,
target_vars_inds, params)
search_space = SearchSpace(parameters=[
RangeParameter(name="lr", lower=1.0e-5, upper=1.0e-1,
parameter_type=ParameterType.FLOAT),
RangeParameter(name="lr_change", lower=0.5, upper=1.0,
parameter_type=ParameterType.FLOAT),
RangeParameter(name="leafes", lower=2, upper=1000,
parameter_type=ParameterType.INT)]
)
experiment = SimpleExperiment(
name = f"weather_lbgm_{dt.datetime.today().strftime('%d-%m-%Y')}",
search_space = search_space,
evaluation_function = mdl.obj_fun,
)
sobol = Models.SOBOL(experiment.search_space)
for i in range(n_random_trials):
def pid(cfg):
env_name = cfg.env.params.name
env = gym.make(env_name)
env.reset()
full_rewards = []
exp_cfg = cfg.experiment
from learn.utils.plotly import hv_characterization
hv_characterization()
def compare_control(env, cfg, save=True):
import torch
from learn.control.pid import PidPolicy
controllers = []
labels = []
metrics = []
# PID baselines
# /Users/nato/Documents/Berkeley/Research/Codebases/dynamics-learn/sweeps/2020-04-14/11-12-02
# from learn.simulate_sac import *
# Rotation policy
sac_policy1 = torch.load(
'/Users/nato/Documents/Berkeley/Research/Codebases/dynamics-learn/outputs/2020-03-24/18-32-26/trial_70000.dat'
)
controllers.append(sac_policy1['policy'])
labels.append("SAC - Rotation")
metrics.append(0)
# Living reward policy
sac_policy2 = torch.load(
'/Users/nato/Documents/Berkeley/Research/Codebases/dynamics-learn/outputs/2020-03-24/18-31-45/trial_35000.dat'
)
controllers.append(sac_policy2['policy'])
labels.append("SAC - Living")
metrics.append(1)
# Square cost policy
# sac_policy2 = torch.load(
# '/Users/nato/Documents/Berkeley/Research/Codebases/dynamics-learn/sweeps/2020-03-25/20-30-47/metric.name=Square,robot=iono_sim/26/trial_40000.dat')
controllers.append(sac_policy2['policy'])
labels.append("SAC - Square")
metrics.append(2)
# un-Optimized PID parameters
pid_params = [[2531.917, 61.358, 33.762], [2531.917, 61.358, 33.762]]
pid = PidPolicy(cfg)
pid.set_params(pid_params)
controllers.append(pid)
labels.append("PID - temp")
metrics.append(0)
controllers.append(pid)
labels.append("PID - temp")
metrics.append(1)
# Optimized PID parameters
pid_params = [[2531.917, 61.358, 3333.762],
[2531.917, 61.358, 3333.762]]
pid = PidPolicy(cfg)
pid.set_params(pid_params)
controllers.append(pid)
labels.append("PID - improved")
metrics.append(2)
from learn.control.mpc import MPController
cfg.policy.mode = 'mpc'
# dynam_model = torch.load(
# '/Users/nato/Documents/Berkeley/Research/Codebases/dynamics-learn/outputs/2020-03-25/10-45-17/trial_1.dat')
dynam_model = torch.load(
'/Users/nato/Documents/Berkeley/Research/Codebases/dynamics-learn/sweeps/2020-03-25/20-30-57/metric.name=Rotation,robot=iono_sim/14/trial_9.dat'
)
mpc = MPController(env, dynam_model['model'], cfg)
controllers.append(mpc)
labels.append("MPC - 1")
metrics.append(0)
controllers.append(mpc)
labels.append("MPC - 2")
metrics.append(1)
controllers.append(mpc)
labels.append("MPC - 3")
metrics.append(2)
import plotly.graph_objects as go
import plotly
colors = [
'#1f77b4', # muted blue
'#ff7f0e', # safety orange
'#2ca02c', # cooked asparagus green
'#d62728', # brick red
'#9467bd', # muted purple
'#8c564b', # chestnut brown
'#e377c2', # raspberry yogurt pink
'#7f7f7f', # middle gray
'#bcbd22', # curry yellow-green
'#17becf' # blue-teal
]
markers = [
"cross",
"circle-open-dot",
"x-open-dot",
"triangle-up-open-dot",
"y-down-open",
"diamond-open-dot",
"hourglass",
"hash",
"star",
"square",
]
m1 = living_reward
m2 = rotation_mat
m3 = squ_cost
eval_metrics = [m1, m2, m3]
metric_names = ["Living", "Rotation", "Square"]
fig = plotly.subplots.make_subplots(
rows=3,
cols=2,
# subplot_titles=["Living", "Rotation", "Square"],
subplot_titles=[
"Pitch",
"Roll",
" ",
" ",
" ",
" ",
],
vertical_spacing=0.03,
horizontal_spacing=0.03,
shared_xaxes=True,
) # go.Figure()
fig_mpc = go.Figure()
fig_sac = go.Figure()
pry = [1, 0, 2]
# state0 = 2*env.reset()
# state0 = env.reset()
state0 = np.array([0, np.deg2rad(15), 0, 0, 0, 0])
for i, (con, lab, m) in enumerate(zip(controllers, labels, metrics)):
print(f"Evaluating controller type {lab}")
_ = env.reset()
env.set_state(np.concatenate((np.zeros(6), state0)))
state = state0
states = []
actions = []
rews = []
done = False
# for t in range(cfg.experiment.r_len + 1):
for t in range(500):
if done:
break
if "SAC" in lab:
with torch.no_grad():
with eval_mode(con):
action = con.select_action(state)
if i < 2:
action = np.array([65535, 65535, 65535, 65535
]) * (action + 1) / 2
else:
action = np.array([3000, 3000, 3000, 3000
]) * (action + 1) / 2
else:
action = con.get_action(state, metric=eval_metrics[m])
states.append(state)
actions.append(action)
state, rew, done, _ = env.step(action)
done = done
states = np.stack(states)
actions = np.stack(actions)
pitch = np.degrees(states[:, pry[0]])
roll = np.degrees(states[:, pry[1]])
# deal with markers
num_mark = np.zeros(len(pitch))
mark_every = 50
m_size = 32
start = np.random.randint(0, int(len(pitch) / 10))
num_mark[start::mark_every] = m_size
if "SAC" in lab:
fig_sac.add_trace(
go.Scatter(
y=pitch,
name=metric_names[m], # legendgroup=lab[:3],
# showlegend=(True if (i % 3 == 0) else False),
line=dict(color=colors[m], width=4),
cliponaxis=False,
mode='lines+markers',
marker=dict(color=colors[m],
symbol=markers[-m],
size=num_mark.tolist())))
elif "MPC" in lab:
fig_mpc.add_trace(
go.Scatter(
y=pitch,
name=metric_names[m], # legendgroup=lab[:3],
# showlegend=(True if (i % 3 == 0) else False),
line=dict(color=colors[m], width=4),
cliponaxis=False,
mode='lines+markers',
marker=dict(color=colors[m],
symbol=markers[-m],
size=num_mark.tolist())))
fig.add_trace(
go.Scatter(
y=pitch,
name=lab[:3] + str(m),
legendgroup=lab[:3],
showlegend=(True if (i % 3 == 0) else False),
line=dict(color=colors[int(i / 3)],
width=2), # mode='lines+markers',
# marker=dict(color=colors[i], symbol=markers[i], size=16)
),
row=m + 1,
col=1)
fig.add_trace(
go.Scatter(
y=roll,
name=lab[:3] + str(m),
legendgroup=lab[:3],
showlegend=(False),
line=dict(color=colors[int(i / 3)],
width=2), # mode='lines+markers',
# marker=dict(color=colors[i], symbol=markers[i], size=16)
),
row=m + 1,
col=2)
fig.update_layout(
title='Comparison of Controllers and Reward Functions',
font=dict(family="Times New Roman, Times, serif",
size=24,
color="black"),
legend_orientation="h",
legend=dict(
x=.6,
y=0.07,
bgcolor='rgba(205, 223, 212, .4)',
bordercolor="Black",
),
# xaxis_title='Timestep',
# yaxis_title='Angle (Degrees)',
plot_bgcolor='white',
width=1600,
height=1000,
# xaxis=dict(
# showline=True,
# showgrid=False,
# showticklabels=True, ),
# yaxis=dict(
# showline=True,
# showgrid=False,
# showticklabels=True, ),
)
fig_sac.update_layout( # title='Comparison of SAC Policies',
font=dict(family="Times New Roman, Times, serif",
size=32,
color="black"),
legend_orientation="h",
legend=dict(
x=.35,
y=0.1,
bgcolor='rgba(205, 223, 212, .4)',
bordercolor="Black",
),
# xaxis_title='Timestep',
# yaxis_title='Angle (Degrees)',
showlegend=False,
plot_bgcolor='white',
width=1600,
height=800,
margin=dict(t=5, r=5),
)
fig_mpc.update_layout( # title='Comparison of MPC Policies',
font=dict(family="Times New Roman, Times, serif",
size=32,
color="black"),
legend_orientation="h",
showlegend=False,
legend=dict(
x=.35,
y=0.1,
bgcolor='rgba(205, 223, 212, .4)',
bordercolor="Black",
# ncol= 2,
),
# xaxis_title='Timestep',
# yaxis_title='Angle (Degrees)',
plot_bgcolor='white',
width=1600,
height=800,
margin=dict(t=5, r=5),
)
reg_color = 'rgba(255,60,60,.15)'
fig_sac.add_trace(
go.Scatter(x=[0, 500],
y=[5, 5],
name='Living Region',
legendgroup='Living Region',
fill='tozeroy',
mode='lines',
fillcolor=reg_color,
line=dict(width=0.0,
color=reg_color))) # fill down to xaxis
fig_sac.add_trace(
go.Scatter(x=[0, 500],
y=[-5, -5],
showlegend=False,
legendgroup='Living Region',
fill='tozeroy',
mode='lines',
fillcolor=reg_color,
line=dict(width=0.0,
color=reg_color))) # fill down to xaxis
fig_mpc.add_trace(
go.Scatter(x=[0, 500],
y=[5, 5],
name='Living Region',
legendgroup='Living Region',
fill='tozeroy',
mode='lines',
fillcolor=reg_color,
line=dict(width=0.0,
color=reg_color))) # fill down to xaxis
fig_mpc.add_trace(
go.Scatter(x=[0, 500],
y=[-5, -5],
showlegend=False,
legendgroup='Living Region',
fill='tozeroy',
mode='lines',
fillcolor=reg_color,
line=dict(width=0.0,
color=reg_color))) # fill down to xaxis
# SOLO
rang_ind = [-20, 20]
fig_sac.update_xaxes(
title_text="Timestep",
range=[0, 500],
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig_sac.update_yaxes(
title_text="Pitch (degrees)",
range=rang_ind,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig_sac.show()
fig_sac.write_image(os.getcwd() + "/compare_sac.pdf")
fig_mpc.update_xaxes(
title_text="Timestep",
range=[0, 500],
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig_mpc.update_yaxes(
title_text="Pitch (degrees)",
range=rang_ind,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig_mpc.show()
fig_mpc.write_image(os.getcwd() + "/compare_mpc.pdf")
# COMPARISON
fig.update_xaxes(
title_text="Timestep",
row=3,
col=1,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_xaxes(
row=2,
col=1,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_xaxes(
row=1,
col=1,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_xaxes(
title_text="Timestep",
row=3,
col=2,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_xaxes(
row=2,
col=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_xaxes(
row=1,
col=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
# fig.update_xaxes(title_text="xaxis 1 title", row=1, col=1)
# fig.update_yaxes(title_text="Roll (Degrees)", row=1, col=1)
rang = [-30, 30]
nticks = 6
fig.update_yaxes(
title_text="Living Rew.",
range=rang,
row=1,
col=1,
nticks=nticks,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_yaxes(
title_text="Rotation Rew.",
range=rang,
row=2,
col=1,
nticks=nticks,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_yaxes(
title_text="Square Cost",
range=rang,
row=3,
col=1,
nticks=nticks,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_yaxes(
range=rang,
row=1,
col=2,
nticks=nticks,
showticklabels=False,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_yaxes(
range=rang,
row=2,
col=2,
nticks=nticks,
showticklabels=False,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_yaxes(
range=rang,
row=3,
col=2,
nticks=nticks,
showticklabels=False,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
print(f"Plotting {len(labels)} control responses")
# save = False
# if save:
# fig.write_image(os.getcwd() + "compare.png")
# else:
# fig.show()
#
# return fig
# compare_control(env, cfg, save=True)
# quit()
plot_results(logx=False, save=True, mpc=False)
quit()
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # Evalutation Function # # # # # # # # # # # # # # # # # # # #
def bo_rollout_wrapper(params, weights=None): # env, controller, exp_cfg):
pid_1 = [params["pitch-p"], params["pitch-i"], params["pitch-d"]]
# pid_1 = [params["roll-p"], params["roll-i"],
# params["roll-d"]] # [params["pitch-p"], params["pitch-i"], params["pitch-d"]]
pid_2 = [params["roll-p"], params["roll-i"], params["roll-d"]]
print(
f"Optimizing Parameters {np.round(pid_1, 3)},{np.round(pid_2, 3)}")
pid_params = [[pid_1[0], pid_1[1], pid_1[2]],
[pid_2[0], pid_2[1], pid_2[2]]]
# pid_params = [[1000, 0, 0], [1000, 0, 0]]
pid = PidPolicy(cfg)
pid.set_params(pid_params)
cum_cost = []
r = 0
fncs = [squ_cost, living_reward, rotation_mat]
mult_rewards = [[] for _ in range(len(fncs))]
while r < cfg.experiment.repeat:
pid.reset()
states, actions, rews, sim_error = rollout(env, pid, exp_cfg)
# plot_rollout(states, actions, pry=[1, 0, 2])
rewards_full = get_rewards(states, actions, fncs=fncs)
for i, vec in enumerate(rewards_full):
mult_rewards[i].append(vec)
# if sim_error:
# print("Repeating strange simulation")
# continue
# if len(rews) < 400:
# cum_cost.append(-(cfg.experiment.r_len - len(rews)) / cfg.experiment.r_len)
# else:
rollout_cost = np.sum(rews) / cfg.experiment.r_len # / len(rews)
# if rollout_cost > max_cost:
# max_cost = rollout_cost
# rollout_cost += get_reward_euler(states[-1], actions[-1])
cum_cost.append(rollout_cost)
r += 1
std = np.std(cum_cost)
cum_cost = np.mean(cum_cost)
# print(f"Cum. Cost {cum_cost / max_cost}")
# print(f"- Mean Cum. Cost / Rew: {cum_cost}, std dev: {std}")
eval = {
"Square": (np.mean(rewards_full[0]), np.std(rewards_full[0])),
"Living": (np.mean(rewards_full[1]), np.std(rewards_full[1])),
"Rotation": (np.mean(rewards_full[2]), np.std(rewards_full[2]))
}
for n, (key, value) in enumerate(eval.items()):
if n == 0:
print(f"- Square {np.round(value, 4)}")
elif n == 1:
print(f"- Living {np.round(value, 4)}")
else:
print(f"- Rotn {np.round(value, 4)}")
return eval
# return cum_cost.reshape(1, 1), std
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
from ax import (
ComparisonOp,
ParameterType,
RangeParameter,
SearchSpace,
SimpleExperiment,
OutcomeConstraint,
)
exp = SimpleExperiment(
name="PID Control Robot",
search_space=SearchSpace([
RangeParameter(
name=f"roll-p",
parameter_type=ParameterType.FLOAT,
lower=1.0,
upper=10000.0,
log_scale=True,
),
# FixedParameter(name="roll-i", value=0.0, parameter_type=ParameterType.FLOAT),
RangeParameter(
name=f"roll-i",
parameter_type=ParameterType.FLOAT,
lower=0,
upper=1000.0,
log_scale=False,
),
RangeParameter(
name=f"roll-d",
parameter_type=ParameterType.FLOAT,
lower=.1,
upper=5000.0,
log_scale=True,
),
RangeParameter(
name=f"pitch-p",
parameter_type=ParameterType.FLOAT,
lower=1.0,
upper=10000.0,
log_scale=True,
),
RangeParameter(
name=f"pitch-d",
parameter_type=ParameterType.FLOAT,
lower=0,
upper=1000.0,
log_scale=False,
),
RangeParameter(
name=f"pitch-i",
parameter_type=ParameterType.FLOAT,
lower=.1,
upper=5000.0,
log_scale=True,
),
# FixedParameter(name="pitch-i", value=0.0, parameter_type=ParameterType.FLOAT),
]),
evaluation_function=bo_rollout_wrapper,
objective_name=cfg.metric.name,
minimize=cfg.metric.minimize,
outcome_constraints=[],
)
from ax.storage.metric_registry import register_metric
from ax.storage.runner_registry import register_runner
class GenericMetric(Metric):
def fetch_trial_data(self, trial):
records = []
for arm_name, arm in trial.arms_by_name.items():
params = arm.parameters
mean, sem = bo_rollout_wrapper(params)
records.append({
"arm_name": arm_name,
"metric_name": self.name,
"mean": mean,
"sem": sem,
"trial_index": trial.index,
})
return Data(df=pd.DataFrame.from_records(records))
class MyRunner(Runner):
def run(self, trial):
return {"name": str(trial.index)}
optimization_config = OptimizationConfig(objective=Objective(
metric=GenericMetric(name=cfg.metric.name),
minimize=cfg.metric.minimize,
), )
register_metric(GenericMetric)
register_runner(MyRunner)
exp.runner = MyRunner()
exp.optimization_config = optimization_config
log.info(f"Running experiment, metric name {cfg.metric.name}")
log.info(f"Running Sobol initialization trials...")
sobol = Models.SOBOL(exp.search_space)
num_search = cfg.bo.random
for i in range(num_search):
exp.new_trial(generator_run=sobol.gen(1))
exp.trials[len(exp.trials) - 1].run()
import plotly.graph_objects as go
gpei = Models.BOTORCH(experiment=exp, data=exp.eval())
objectives = ["Living", "Square", "Rotation"]
def plot_all(model, objectives, name="", rend=False):
for o in objectives:
plot = plot_contour(
model=model,
param_x="roll-p",
param_y="roll-d",
metric_name=o,
)
plot[0]['layout']['title'] = o
data = plot[0]['data']
lay = plot[0]['layout']
for i, d in enumerate(data):
if i > 1:
d['cliponaxis'] = False
fig = {
"data": data,
"layout": lay,
}
go.Figure(fig).write_image(name + o + ".png")
if rend: render(plot)
plot_all(gpei, objectives, name="Random fit-")
num_opt = cfg.bo.optimized
for i in range(num_opt):
log.info(f"Running GP+EI optimization trial {i + 1}/{num_opt}...")
# Reinitialize GP+EI model at each step with updated data.
batch = exp.new_trial(generator_run=gpei.gen(1))
gpei = Models.BOTORCH(experiment=exp, data=exp.eval())
if ((i + 1) % 10) == 0:
plot_all(gpei,
objectives,
name=f"optimizing {str(i + 1)}-",
rend=False)
from ax.plot.exp_utils import exp_to_df
best_arm, _ = gpei.gen(1).best_arm_predictions
best_parameters = best_arm.parameters
log.info(f"Best parameters {best_parameters}")
experiment_log = {
"Exp": exp_to_df(exp=exp),
"Cfg": cfg,
"Best_param": best_parameters,
}
log.info("Printing Parameters")
log.info(exp_to_df(exp=exp))
save_log(cfg, exp, experiment_log)
fig_learn = plot_learning(exp, cfg)
fig_learn.write_image("learning" + ".png")
fig_learn.show()
plot_all(gpei, objectives, name=f"FINAL -", rend=True)
def search_space():
"""
Defines the network search space parameters and returns the search
space object
Returns
------
Search space object
"""
params = []
##### RNN BLOCKS ######################################################################
params.append(
RangeParameter(name="rnn_layers",
parameter_type=ParameterType.INT,
lower=1,
upper=5))
params.append(
RangeParameter(name="neurons_layers",
parameter_type=ParameterType.INT,
lower=8,
upper=512))
params.append(
RangeParameter(name="rnn_dropout",
parameter_type=ParameterType.FLOAT,
lower=0.1,
upper=0.5))
params.append(
RangeParameter(name="cell_type",
parameter_type=ParameterType.INT,
lower=0,
upper=1))
#######################################################################################
### FC BLOCKS ########################################################################
params.append(
RangeParameter(name="fc_layers",
lower=0,
upper=1,
parameter_type=ParameterType.INT))
params.append(
RangeParameter(
name="neurons_fc_layer_1",
lower=10,
upper=128,
parameter_type=ParameterType.INT,
))
#### SPLIT SEUENCES ##################################################
params.append(
RangeParameter(name="max_len",
lower=50,
upper=250,
parameter_type=ParameterType.INT))
###### MANDATORY PARAMETERS ############################################
params.append(
RangeParameter(
name="learning_rate",
lower=0.0001,
upper=0.01,
parameter_type=ParameterType.FLOAT,
))
params.append(
RangeParameter(
name="learning_gamma",
lower=0.9,
upper=0.99,
parameter_type=ParameterType.FLOAT,
))
params.append(
RangeParameter(name="learning_step",
lower=1,
upper=10,
parameter_type=ParameterType.INT))
params.append(
RangeParameter(
name="prune_threshold",
lower=0.05,
upper=0.9,
parameter_type=ParameterType.FLOAT,
))
params.append(
RangeParameter(name="batch_size",
lower=2,
upper=128,
parameter_type=ParameterType.INT))
########################################################################
search_space = SearchSpace(parameters=params)
return search_space
def optimize(self):
SOBOL_TRIALS = 75
gpei_list = [50, 30, 10, 0]
parameters = None
for gpei_trials in gpei_list:
try:
search_space = SearchSpace(parameters=[
ChoiceParameter(name='risk_function',
values=['polynomial', 'exponential'],
parameter_type=ParameterType.STRING),
RangeParameter(name='alpha_poly',
lower=1.0,
upper=5.0,
parameter_type=ParameterType.FLOAT),
RangeParameter(name='alpha_exp',
lower=0.0,
upper=1.0,
parameter_type=ParameterType.FLOAT),
RangeParameter(name='exp_threshold',
lower=1.0,
upper=10.0,
parameter_type=ParameterType.FLOAT)
])
experiment = SimpleExperiment(
name='risk_function_parametrisation',
search_space=search_space,
evaluation_function=self.evaluate_parameterization,
objective_name='par10',
minimize=True)
sobol = Models.SOBOL(experiment.search_space)
for _ in range(SOBOL_TRIALS):
experiment.new_trial(generator_run=sobol.gen(1))
best_arm = None
for _ in range(gpei_trials):
gpei = Models.GPEI(experiment=experiment,
data=experiment.eval())
generator_run = gpei.gen(1)
best_arm, _ = generator_run.best_arm_predictions
experiment.new_trial(generator_run=generator_run)
parameters = best_arm.parameters
break
except:
print('GPEI Optimization failed')
if gpei_trials == 0:
# choose expectation if optimization failed for all gpei_trial values
# exp thresholds are dummy variables
parameters = {
'risk_function': 'polynomial',
'alpha_poly': 1.0,
'alpha_exp': 1.0,
'exp_threshold': 1.0
}
else:
continue
return self.resolve_risk_function(parameters['risk_function'],
parameters['alpha_poly'],
parameters['alpha_exp'],
parameters['exp_threshold'])
args = parser.parse_args()
dset = args.dset
lr = args.lr
decay = args.decay
warmups = args.warmups
eps = args.eps
init_trials = args.init_trials
opt_trials = args.opt_trials
n_epochs = args.n_epochs
type = args.type
# Search space
transformer_search_space = SearchSpace(parameters=[
RangeParameter(name='lr',
parameter_type=ParameterType.FLOAT,
lower=min(lr),
upper=max(lr),
log_scale=False),
RangeParameter(name='decay',
parameter_type=ParameterType.FLOAT,
lower=min(decay),
upper=max(decay),
log_scale=False),
RangeParameter(name='warmups',
parameter_type=ParameterType.FLOAT,
lower=min(warmups),
upper=max(warmups),
log_scale=False),
RangeParameter(name='eps',
parameter_type=ParameterType.FLOAT,
lower=min(eps),
def __init__(self, serialized_filepath=None):
self.serialized_filepath = serialized_filepath
if serialized_filepath is not None and os.path.exists(
serialized_filepath):
with open(serialized_filepath, "r") as f:
serialized = json.load(f)
other = CoreAxClient.from_json_snapshot(serialized)
self.__dict__.update(other.__dict__)
else:
parameters = [
RangeParameter("num_epochs",
ParameterType.INT,
lower=30,
upper=200),
RangeParameter("log2_batch_size",
ParameterType.INT,
lower=5,
upper=8),
RangeParameter("lr",
ParameterType.FLOAT,
lower=1e-5,
upper=0.3,
log_scale=True),
RangeParameter("gamma_prewarmup",
ParameterType.FLOAT,
lower=0.5,
upper=1.0),
RangeParameter("gamma_warmup",
ParameterType.FLOAT,
lower=0.5,
upper=1.0),
RangeParameter("gamma_postwarmup",
ParameterType.FLOAT,
lower=0.5,
upper=0.985),
RangeParameter("reg_warmup_start_epoch",
ParameterType.INT,
lower=1,
upper=200),
RangeParameter("reg_warmup_end_epoch",
ParameterType.INT,
lower=1,
upper=200),
# Parameter constraints not allowed on log scale
# parameters. So implement the log ourselves.
RangeParameter("log_reg_factor_start",
ParameterType.FLOAT,
lower=math.log(1e-4),
upper=math.log(1.0)),
RangeParameter("log_reg_factor_end",
ParameterType.FLOAT,
lower=math.log(0.1),
upper=math.log(10.0)),
]
pm = {p.name: p for p in parameters}
search_space = SearchSpace(
parameters=parameters,
parameter_constraints=[
# reg_warmup_start_epoch <= reg_warmup_end_epoch
OrderConstraint(pm["reg_warmup_start_epoch"],
pm["reg_warmup_end_epoch"]),
# reg_warmup_end_epoch <= num_epochs
OrderConstraint(pm["reg_warmup_end_epoch"],
pm["num_epochs"]),
# log_reg_factor_start <= log_reg_factor_end
OrderConstraint(pm["log_reg_factor_start"],
pm["log_reg_factor_end"]),
])
optimization_config = OptimizationConfig(objective=MultiObjective(
metrics=[
Metric(name="neg_log_error", lower_is_better=False),
Metric(name="neg_log_num_nonzero_weights",
lower_is_better=False)
],
minimize=False,
), )
generation_strategy = choose_generation_strategy(
search_space,
enforce_sequential_optimization=False,
no_max_parallelism=True,
num_trials=NUM_TRIALS,
num_initialization_trials=NUM_RANDOM)
super().__init__(experiment=Experiment(
search_space=search_space,
optimization_config=optimization_config),
generation_strategy=generation_strategy)
def example_f33(x):
# Distance from a multiple of 33
mod33 = x - (x // 33) * 33
if mod33 > 33 // 2:
return 33 - mod33
else:
return mod33
NUM_TRIALS = 20
NUM_RANDOM = 10
search_space = SearchSpace(parameters=[
RangeParameter("x", ParameterType.FLOAT, lower=12.2, upper=602.2),
])
optimization_config = OptimizationConfig(
objective=MultiObjective(
metrics=[
# Currently MultiObjective doesn't work with lower_is_better=True.
# https://github.com/facebook/Ax/issues/289
Metric(name="neg_distance17", lower_is_better=False),
Metric(name="neg_distance33", lower_is_better=False)
],
minimize=False,
), )
generation_strategy = choose_generation_strategy(
search_space, num_trials=NUM_TRIALS, num_initialization_trials=NUM_RANDOM)