def tune_SVM(dataset,
svm_parameters=svm_parameters,
is_marker=False,
seed=0,
total_trials=15):
"""
function to get baseline for standalone SVM
"""
#set a seed so that splits are the same across different model tests
np.random.seed(seed)
#make data splits
train_df, test_df = make_split(dataset)
#for rf, go straight to k-fold
train_dataset = MicroDataset(train_df, is_marker=is_marker)
X = train_dataset.matrix.detach().numpy()
y = train_dataset.y.detach().numpy()
def evaluation_func(p):
#the function used for the hyperparameter tuning
model = SVC(probability=True)
[setattr(model, a, np.power(2., q)) for a, q in p.items()]
roc, model = run_training(X, y, model=model)
auc_val = auc(roc[0], roc[1])
if math.isnan(auc_val):
auc_val = 0
return (auc_val)
best_parameters, best_values, experiment, model = optimize(
parameters=svm_parameters,
evaluation_function=evaluation_func,
total_trials=total_trials,
minimize=False)
#make a model with the best parameters
best_classifier = SVC(probability=True)
[
setattr(best_classifier, a, np.power(2., q))
for a, q in best_parameters.items()
]
# fit it on the full train, valid data
best_classifier.fit(X, y)
# Get AUC for the full setup on the test set
test_dataset = MicroDataset(test_df, is_marker=is_marker)
test_preds = best_classifier.predict_proba(
test_dataset.matrix.detach().numpy())
test_roc = roc_curve(test_dataset.y.detach().numpy(), test_preds[:, 1])
best_val = auc(test_roc[0], test_roc[1])
data_name = dataset.index[0]
if is_marker == False:
data_type = 'Abundance'
else:
data_type = 'Marker'
return (['SVM', data_type, data_name, best_parameters, best_val])
def ax_cube(objective, n_trials, n_dim, with_count=False, method=None):
global feval_count
feval_count = 0
rt = get_logger('ax')
rt.setLevel(CRITICAL)
def evaluation_func(prms):
global feval_count
feval_count += 1
return objective([prms["u" + str(i)] for i in range(n_dim)])
parameters = [{
"name": "u" + str(i),
"type": "range",
"bounds": [0.0, 1.0],
} for i in range(n_dim)]
best_parameters, best_values, experiment, model = optimize(
parameters=parameters,
evaluation_function=evaluation_func,
minimize=True,
total_trials=n_trials)
best_x = [best_parameters['u' + str(i)] for i in range(n_dim)]
best_val = best_values[0]['objective']
return (best_val, best_x, feval_count) if with_count else (best_val,
best_x)
def ax_cube(objective, scale, n_trials):
def evaluation_func(prms):
return objective([prms["u1"], prms["u2"], prms["u3"]])[0]
best_parameters, best_values, experiment, model = optimize(
parameters=[
{
"name": "u1",
"type": "range",
"bounds": [-scale, scale],
},
{
"name": "u2",
"type": "range",
"bounds": [-scale, scale],
},
{
"name": "u3",
"type": "range",
"bounds": [-scale, scale],
},
],
# Booth function
evaluation_function=evaluation_func,
minimize=True,
total_trials=n_trials)
return best_values[0]['objective']
def evaluate(self, error_yielding_function):
random_seed = self.total_trials
best_parameters, values, experiment, model = ax.optimize(
parameters=[
{
'name': 'beta1',
'type': 'choice',
'values': [0.90, 0.95]
},
{
'name': 'lr',
'type': 'range',
'bounds': [0.0008, 0.003]
},
{
'name': 'N_sma_threshold',
'type': 'choice',
'values': [4, 5]
},
{
'name': 'k',
'type': 'choice',
'values': [6, 8]
},
],
evaluation_function=error_yielding_function,
objective_name='accuracy',
total_trials=self.total_trials,
random_seed=random_seed)
return best_parameters
def ax_train(rcml, model_params: dict, config_params: dict):
# ----------------------------------------------------------------------------------------------------
# cross-validation folds
# ----------------------------------------------------------------------------------------------------
global_best_model = None
global_best_test_accuracy = 0
parameters = [
{
"name": "max_depth",
"type": "range",
"bounds": [9, 17],
"parameter_type": ParameterType.INT
},
{
"name": "max_features",
"type": "range",
"bounds": [0.20, 0.6],
"parameter_type": ParameterType.FLOAT
},
{
"name": "n_estimators",
"type": "range",
"bounds": [100, 200],
"parameter_type": ParameterType.INT
}
]
best_parameters, best_values, experiment, model = optimize(
parameters=parameters,
evaluation_function=lambda params: ax_train_proxy(model_params=model_params,
config_params=config_params,
ax_params=params),
minimize=False,
total_trials=10,
objective_name='accuracy',
)
print("Ax Optimization Results:")
print(best_parameters)
print(best_values)
# save model
# rcml.save_best_model(global_best_model)
# ----------------------------------------------------------------------------------------------------
# TODO: baseline [ single-node - multi-GPU w/ dask ]
# ----------------------------------------------------------------------------------------------------
return global_best_model, global_best_test_accuracy
def veritize(self):
total_trials = (DashVerum.v_resp['metric']['num_trials']
if DashVerum.v_resp['metric']['num_trials'] is not None
else 2)
best_parameters, best_values, experiment, model = optimize(
parameters=self.init_parameters(),
evaluation_function=self.evaluation_fn,
minimize=DashVerum.v_resp['metric']['minimize'],
total_trials=total_trials)
if DashVerum.v_resp['return']:
try:
DashVerum.learn.model.ps = best_parameters['dropout_ps']
except:
pass
# try: DashVerum.learn.model.wd = best_parameters['weight_decay']
# except: pass
try:
DashVerum.learn.model.use_bn = best_parameters['use_bn']
except:
pass
return_list = [DashVerum.learn]
try:
return_list.append(best_parameters['learning_rate'])
except:
return_list.append(
DashVerum.v_resp['learning_rate']['default'])
try:
return_list.append(best_parameters['num_epochs'])
except:
return_list.append(DashVerum.v_resp['num_epochs']['default'])
momentum_list = []
try:
momentum_list.append(best_parameters['momentum0'])
except:
momentum_list.append(DashVerum.v_resp['momentum0']['default'])
try:
momentum_list.append(best_parameters['momentum1'])
except:
momentum_list.append(DashVerum.v_resp['momentum1']['default'])
return_list.append(tuple(momentum_list))
return tuple(return_list)
else:
print(json.dumps(best_parameters, indent=4))
def optimize(self, **kwargs):
"""
:param kwargs: kwargs for ax.optimize
"""
def evaluate(kwargs: Dict[str, float]):
run_offseason = self._offseason_runner_builder(**kwargs)
run_league(self._league, run_offseason)
return calc_brier_score(self._league)
best_parameters, best_values, experiment, model = optimize(
parameters=[p.to_ax() for p in self._params.values()],
evaluation_function=evaluate,
minimize=True,
**kwargs,
)
for name, best_value in best_parameters.items():
self._params[name] = self._params[name].update_value(best_value)
return best_parameters, best_values, experiment, model
def tune_MAML_FFNN(datasets, is_marker=False, seed=0, total_trials=15):
#set a seed so that splits are the same across different model tests
np.random.seed(seed)
#make data splits
splits = [make_split(df) for df in datasets]
#do a train/val/test split for the first dataset
train, test = make_split(datasets[0])
train, valid = make_split(train)
splits[0] = (train, valid)
trains = [s[0] for s in splits]
vals = [s[1] for s in splits]
#build evaluation function
eval_func = build_MAML_FFNN_eval_func(train,
valid,
test,
all_trains=trains,
all_valids=vals,
is_marker=is_marker)
#selectthe parameters based on the model_name
parameters = MAML_FFNN_parameters
#optimize hyperparapeter with Facebook's ax tuning
best_parameters, best_values, experiment, model = optimize(
parameters=parameters,
evaluation_function=eval_func,
total_trials=total_trials,
minimize=False)
data_name = train.dataset.values[0]
if is_marker == False:
data_type = 'Abundance'
else:
data_type = 'Marker'
model_name = 'MAML_FFNN'
return ([model_name, data_type, data_name, best_parameters, best_values])
def svm_eval(X, y, svm_hyperparams, n_folds=5, total_trials=10):
# find the best hyperamas vis k-fold cross-validation
def evaluation_func(p):
#the function used for the hyperparameter tuning
model = SVC(probability=True)
[setattr(model, a, np.power(2., q)) for a, q in p.items()]
roc, model = run_training(X, y, n_folds=n_folds, model=model)
auc_val = auc(roc[0], roc[1])
if math.isnan(auc_val):
auc_val = 0
return (auc_val)
best_parameters, best_values, experiment, model = optimize(
parameters=svm_hyperparams,
evaluation_function=evaluation_func,
total_trials=total_trials,
minimize=False)
return (best_parameters)
def rf_eval(X, y, rf_hyperparams, n_folds=5, total_trials=10):
# find the best hyperamas vis k-fold cross-validation
def evaluation_func(p):
#the function used for the hyperparameter tuning
model = RandomForestClassifier()
[setattr(model, a, q) for a, q in p.items()]
roc, model = run_training(X, y, n_folds=n_folds, model=model)
auc_val = auc(roc[0], roc[1])
if math.isnan(auc_val):
auc_val = 0
return (auc_val)
best_parameters, best_values, experiment, model = optimize(
parameters=rf_hyperparams,
evaluation_function=evaluation_func,
total_trials=total_trials,
minimize=False)
return (best_parameters)
def dont_test_intro_example():
""" https://ax.dev/ """
best_parameters, best_values, experiment, model = optimize(
parameters=[
{
"name": "x1",
"type": "range",
"bounds": [-10.0, 10.0],
},
{
"name": "x2",
"type": "range",
"bounds": [-10.0, 10.0],
},
],
# Booth function
evaluation_function=lambda p: (p["x1"] + 2 * p["x2"] - 7) ** 2 + (2 * p["x1"] + p["x2"] - 5) ** 2,
minimize=True,
)
return best_values
def ax_cube(objective,scale, n_trials, n_dim, with_count=False):
global feval_count
feval_count = 0
def evaluation_func(prms):
global feval_count
feval_count += 1
return objective([prms["u"]+str(i) for i in range(n_dim)])[0]
parameters = [ {
"name": "u"+str(i),
"type": "range",
"bounds": [-scale, scale],
} for i in range(n_dim)]
best_parameters, best_values, experiment, model = optimize( parameters=parameters,
# Booth function
evaluation_function=evaluation_func,
minimize=True,
total_trials=n_trials)
return (best_values[0]['objective'], feval_count) if with_count else best_values[0]['objective']
def optimize(self):
# ax.optimizeをコールする
# ax.optimizeは最適化が終了するまで復帰して来ない
# ax.optimize内でevaluation_functionが試行回数分コールされる
best_parameters, best_values, experiment, model = optimize(
parameters=[ # 最適化対象パラメータの定義
{
"name": "x1",
"type": "range",
"bounds": [-10.0, 10.0],
},
{
"name": "x2",
"type": "range",
"bounds": [-10.0, 10.0],
},
],
# 評価関数（オリジナルコード）
# evaluation_function=lambda p: (p["x1"] + 2*p["x2"] - 7)**2 + (2*p["x1"] + p["x2"] - 5)**2,
evaluation_function=self.evaluation_function, # 評価関数
objective_name="score", # 評価指標の名前
minimize=True, # 最適化＝最小化
)
# best_parameters contains {'x1': 1.02, 'x2': 2.97};
# the global min is (1, 3)
# ベストパラメータ＝最適解に最も近づいた値
print(f"best_parameters = {best_parameters}")
# その他指標
means, covariances = best_values
print(f"means = {means}")
if self.cb_end:
# 終了コールバック：Unityアプリへベストパラメータを通知する
x1 = best_parameters['x1']
x2 = best_parameters['x2']
x = np.array([x1, x2])
self.cb_end(x)
def optimize(self, func, bounds, args=(), x0=None, **kwargs):
def optimizing_func(x):
return func(list(x.values()), *args)
parameters = [{
"name": f"x{i}",
"type": "range",
"bounds": [float(b[0]), float(b[1])],
"value_type": "float",
"log_scale": self._log_scale
} for i, b in enumerate(bounds)]
best_parameters, best_values, experiment, model = optimize(
experiment_name="optimization",
objective_name="seismic_metrics",
evaluation_function=optimizing_func,
parameters=parameters,
total_trials=self._num_evals,
minimize=True
)
return list(best_parameters.values()), {}
def ax_optimize(scoring_function, tunable_hyperparameters, iterations):
"""Construct and run a full optimization loop.
Convert the hyperparameters in to the accepted format by ``Ax`` and adapt the scoring
function to pass the configuration as kwargs.
Args:
scoring_function (function):
A function that performs scoring over params.
tunable_hyperparameters (dict):
A python dict with hyperparameters.
iterations (int):
Number of tuning iterations to perform.
"""
parameters = convert_hyperparameters(tunable_hyperparameters)
evaluation_function = adapt_scoring_function(scoring_function)
best_params = optimize(parameters=parameters,
evaluation_function=evaluation_function,
total_trials=iterations,
minimize=False)[0]
return evaluation_function(best_params)
best_parameters, best_values, experiment, model = optimize(
parameters=[
{
"name": "decay_start",
"type": "range",
"bounds": [100, 40000],
"value_type": "int",
"log_scale": True
},
{
"name": "lr_A",
"type": "range",
"bounds": [1e-5, 500e-5],
"value_type": "float",
"log_scale": True
},
{
"name": "lr_B",
"type": "range",
"bounds": [100.0, 40000.0],
"value_type": "float",
"log_scale": True
},
{
"name": "epochs_between_updates",
"type": "range",
"bounds": [1, 6],
"value_type": "int",
"log_scale": False
},
{
"name": "decoder_rnn_dim",
"type": "range",
"bounds": [768, 1280],
"value_type": "int",
"log_scale": True
},
{
"name": "p_attention_dropout",
"type": "range",
"bounds": [0, 0.5],
"value_type": "float",
"log_scale": False
},
{
"name": "p_decoder_dropout",
"type": "range",
"bounds": [0, 0.5],
"value_type": "float",
"log_scale": False
},
{
"name": "attention_rnn_dim",
"type": "range",
"bounds": [1024, 4096],
"value_type": "int",
"log_scale": True
},
{
"name": "attention_dim",
"type": "range",
"bounds": [64, 512],
"value_type": "int",
"log_scale": True
},
{
"name": "attention_location_n_filters",
"type": "range",
"bounds": [16, 128],
"value_type": "int",
"log_scale": True
},
{
"name": "speaker_embedding_dim",
"type": "range",
"bounds": [64, 2048],
"value_type": "int",
"log_scale": True
},
],
# Booth function
total_trials=15,
evaluation_function=train_experiment,
minimize=False,
objective_name='validation_average_max_attention_weight',
)
def tune_shared_encoder(datasets,
model_name='DAE',
is_marker=False,
seed=0,
total_trials=15):
if model_name == 'SAE':
# we only have 10 possible hyperparam combos for SAE -- this is just gridsearch
total_trials = 10
#set a seed so that splits are the same across different model tests
np.random.seed(seed)
#make data splits
#doing train/valid/test splits across all datasets
# ==> note - we don't really need test split for t
splits = [make_split(df) for df in datasets]
#splits = [(make_split(a[0]), a[1]) for a in splits ]
#redo the split for the dataset of interest (the one at idx 0)
# so we include a test set
train, test = make_split(datasets[0])
train, valid = make_split(train)
splits[0] = (train, valid)
trains = pd.concat([s[0] for s in splits], axis=0).reset_index(drop=True)
vals = pd.concat([s[1] for s in splits], axis=0).reset_index(drop=True)
#tests = pd.concat( [s[1] for s in splits], axis=0 ).reset_index(drop=True)
# for i in range(1, len(trains)):
# # we don't need the test set for the dataset's we aren't testing on
# trains[i] = pd.concat( [trains[i],tests[i]], axis = 0 )
eval_func = build_sharing_encoder_eval_func(train,
valid,
test,
all_trains=trains,
all_valids=vals,
model_name=model_name,
is_marker=is_marker)
#selectthe parameters based on the model_name
if model_name == 'DAE':
parameters = DAE_parameters
if model_name == 'SAE':
parameters = SAE_parameters
if model_name == 'VAE':
parameters = VAE_parameters
#optimize hyperparapeter with Facebook's ax tuning
best_parameters, best_values, experiment, model = optimize(
parameters=parameters,
evaluation_function=eval_func,
total_trials=total_trials,
minimize=False)
data_name = train.dataset.values[0]
if is_marker == False:
data_type = 'Abundance'
else:
data_type = 'Marker'
return ([model_name, data_type, data_name, best_parameters, best_values])
best_parameters, best_values, experiment, model = optimize(
parameters=[
{
"name": "number_of_twisted_points",
"type": "range",
"value_type": "int",
"bounds": [2, 6],
},
{
"name": "from_last_twisted_point_to_forward",
"type": "range",
"value_type": "int",
"bounds": [2, 5],
},
{
"name": "direction_error",
"type": "fixed",
"value_type": "float",
"value": 10 / 100
},
{
"name": "consecutive_error",
"type": "range",
"value_type": "float",
"bounds": [0.5 / 100, 25 / 100],
},
{
"name": "ema_fast_v",
"type": "choice",
"value_type": "int",
"values": [9, 14, 12, 15, 17],
},
{
"name": "ema_slow_v",
"type": "choice",
"value_type": "int",
"values": [21, 28, 32, 45, 50],
},
{
"name": "prediction_index",
"type": "choice",
"value_type": "int",
"values": [4, 5, 6, 7, 8, 9, 10],
},
],
# Booth function
evaluation_function=lambda v: evaluate_moving_avg(
number_of_twisted_points=int(v["number_of_twisted_points"]),
from_last_twisted_point_to_forward=int(v[
"from_last_twisted_point_to_forward"]),
direction_error=float(v["direction_error"]),
consecutive_error=float(v["consecutive_error"]),
ema_fast_v=int(v["ema_fast_v"]),
ema_slow_v=int(v["ema_slow_v"]),
prediction_index=int(v["prediction_index"]),
),
minimize=False,
)
best_parameters, best_values, experiment, model = optimize(
parameters=[
{
"name": "factory_s",
"type": "range",
"bounds": [0.0, 30.0],
},
{
"name": "factory_Q",
"type": "range",
"bounds": [0.0, 30.0],
},
{
"name": "w1_s",
"type": "range",
"bounds": [0.0, 20.0],
},
{
"name": "w1_Q",
"type": "range",
"bounds": [0.0, 20.0],
},
{
"name": "w2_s",
"type": "range",
"bounds": [0.0, 20.0],
},
{
"name": "w2_Q",
"type": "range",
"bounds": [0.0, 20.0],
},
{
"name": "w3_s",
"type": "range",
"bounds": [0.0, 20.0],
},
{
"name": "w3_Q",
"type": "range",
"bounds": [0.0, 20.0],
},
{
"name": "w4_s",
"type": "range",
"bounds": [0.0, 20.0],
},
{
"name": "w4_Q",
"type": "range",
"bounds": [0.0, 20.0],
},
],
evaluation_function=f_policy,
minimize=False,
total_trials=200,
)
def main():
# TODO model 1 numexpochs 30, model 2 - maxxxx
parser = ArgumentParser()
parser.add_argument('--skip_train', help='if skip train', action='store_true', default=False)
parser.add_argument('--model_path', help='if skip train - path model to load', type=str,
default="models/model1.pth")
parser.add_argument('--num_epochs', type=int, default=30)
parser.add_argument('--msg', help='msg to write in log file', type=str, default='')
parser.add_argument('--n_epochs_stop', help='early stopping in training', type=int, default=10)
parser.add_argument('--comp', action='store_true', default=False)
parser.add_argument('--total_trails', type=int, default=70)
parser.add_argument('--debug', action='store_true', default=False)
parser.add_argument('--search_hyperparams', action='store_true', default=False)
args = parser.parse_args()
# Gets or creates a logger
logging.config.fileConfig('logging.conf')
logger = logging.getLogger(__name__)
# Data paths
data_dir = "Data/"
if args.debug:
path_train = data_dir + "small_train.labeled"
path_test = data_dir + "small_test.labeled"
else:
path_train = data_dir + "train.labeled"
path_test = data_dir + "test.labeled"
logger.debug(f"Starting train: {path_train} test:{path_test}")
if args.search_hyperparams:
best_parameters, best_values, experiment, model = optimize(
parameters=[
{
"name": "accumulate_grad_step",
"type": "choice",
"is_numeric": False,
"values": [2, 5, 10, 20],
},
{
"name": "optimizer_method",
"type": "choice",
"is_numeric": False,
"values": ["{'optim':optim.Adam, 'lr':0.001, 'betas':(0.9, 0.999)}",
"{'optim':optim.Adam, 'lr':0.01, 'betas':(0.9, 0.9)}",
"{'optim':optim.Adam, 'lr':0.001, 'betas':(0.9, 0.9)}",
"{'optim':optim.Adam, 'lr':0.0005, 'betas':(0.9, 0.9)}",
"{'optim':optim.SGD, 'lr':0.01}",
"{'optim':optim.SGD, 'lr':0.1}",
"{'optim':optim.SGD, 'lr':0.001}",
"{'optim':optim.SGD, 'lr':1.0}",
"{'optim':optim.AdamW, 'lr':0.001, 'betas':(0.9, 0.999)}",
"{'optim':optim.AdamW, 'lr':0.002, 'betas':(0.9, 0.9)}",
"{'optim':optim.AdamW, 'lr':0.0005, 'betas':(0.9, 0.9)}",
"{'optim':optim.AdamW, 'lr':0.001, 'betas':(0.9, 0.9)}",
"{'optim':optim.Adadelta, 'lr':1}",
"{'optim':optim.Adadelta, 'lr':2}"],
},
{
"name": "lstm_hidden_dim",
"type": "choice",
"is_numeric": False,
"values": [125, 200, 300, 400, 0],
},
{
"name": "word_embedding_name_or_size_and_freeze_flag",
"type": "choice",
"is_numeric": False,
"values": ["('glove.6B.50d', True)",
"('glove.6B.50d', False)",
"('glove.6B.100d', True)",
"('glove.6B.100d', False)",
"('glove.6B.200d', True)",
"('glove.6B.200d', False)",
"('glove.6B.300d', True)",
"('glove.6B.300d', False)",
"('glove.840B.300d', True)",
"('glove.840B.300d', False)",
"('fasttext.en.300d', True)",
"('fasttext.en.300d', False)",
"('25', False)",
"('50', False)",
"('100', False)",
"('200', False)",
"('300', False)"]
},
{
"name": "tag_embedding_dim",
"type": "choice",
"is_numeric": False,
"values": [5, 10, 25, 50, 150],
},
{
"name": "mlp_hidden_dim",
"type": "choice",
"is_numeric": False,
"values": [50, 100, 200, 300, 400, 500],
},
{
"name": "bilstm_layers",
"type": "choice",
"is_numeric": False,
"values": [1, 2, 3, 4]
},
{
"name": "dropout_alpha",
"type": "choice",
"is_numeric": False,
"values": [0.0, 0.05, 0.1, 0.25]
},
{
"name": "lstm_dropout",
"type": "choice",
"is_numeric": False,
"values": [0.0, 0.15, 0.3]
},
{
"name": "activation",
"type": "choice",
"is_numeric": False,
"values": ["nn.Tanh", "nn.ReLU", "nn.Sigmoid", "nn.LeakyReLU"]
},
{
"name": "min_freq",
"type": "choice",
"is_numeric": False,
"values": [1, 2, 3, 5]
},
{
"name": "mlp_dropout",
"type": "choice",
"is_numeric": False,
"values": [0.0, 0.15, 0.3]
},
],
evaluation_function=lambda p: optimization_wrapper(args, logger, path_train, path_test, p),
minimize=False,
total_trials=args.total_trails,
objective_name="UAS accuracy",
)
logger.debug(f"{best_parameters},{best_values[0]}")
else:
optimization_wrapper(args, logger, path_train, path_test, parameters_advanced_model)
best_parameters, best_values, _, _ = optimize(
# https://ax.dev/docs/core.html#search-space-and-parameters
parameters=[
{"name": "INITIAL_LR",
"type": "range",
"bounds": [5e-4, 1e-3]
},
{"name": "GAMMA",
"type": "range",
"bounds": [0.8, 0.999]
},
{"name": "FILTERS",
"type": "choice",
"values": [
'(128, 128, 256)',
'(64, 128, 256)',
'(128, 256, 256)',
]
},
{"name": "OUTPUT_FC_HIDDEN_SIZES",
"type": "choice",
"values": ['(256, 256)', '(128, 256)', '(512, 256)', '(1024, 256)', '(2048, 256)', '(2048,)', ]
},
{
"name": "KERNEL_SIZES",
"type": "choice",
"values": [
"('(1, 8, 8)', '(1, 4, 4)', '(4, 3, 3)')",
"('(1, 8, 8)', '(1, 4, 4)', '(4, 1, 1)')",
"('(1, 8, 8)', '(1, 3, 3)', '(4, 3, 3)')",
"('(1, 8, 8)', '(1, 5, 5)', '(4, 3, 3)')",
],
},
{
"name": "STRIDE_SIZES",
"type": "choice",
"values": [
"('(1, 3, 3)', '(1, 3, 3)', '(1, 3, 3)')",
"('(1, 4, 4)', '(1, 2, 2)', '(1, 1, 1)')",
"('(1, 5, 5)', '(1, 3, 3)', '(1, 3, 3)')",
"('(1, 4, 4)', '(1, 2, 2)', '(1, 3, 3)')",
],
},
{"name": "OUTPUT_HIDDEN_DROPOUT",
"type": "choice",
"values": [0, 0.1, 0.2],
"value_type": 'float',
},
{"name": "CATEGORICAL",
"type": "choice",
"values": [True, False]
},
{"name": "SUPPORT_RANGE",
"type": "choice",
"values": ['(-10, 20)', '(-10, 10)']
},
{"name": "NOISY",
"type": "choice",
"values": [True, False]
},
{"name": "DUELING",
"type": "choice",
"values": [True, False]
},
# {"name": "GRAYSCALE",
# "type": "choice",
# "values": [True, False]
# },
],
evaluation_function=visual_banana_tuning,
minimize=False,
arms_per_trial=1,
total_trials=NUM_TRIALS
)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-config", type=str, required=True)
parser.add_argument("-mode", type=str, default="train")
parser.add_argument("-hyper", action='store_true')
parser.add_argument("-es", action='store_true')
args = parser.parse_args()
global params
params = runpy.run_path(args.config).get('model_params', None)
pl.seed_everything(params["random_seed"])
if params["file_type"] == "sdf":
loader = SDFDataLoader(params["train_data_file"],
label_props=params["label_cols"])
else:
loader = SMILESDataLoader(params["train_data_file"],
smiles_col=params["smiles_col"],
label_props=params["label_cols"])
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_val = train_test_split(
loader.mols,
loader.labels_list,
shuffle=True,
train_size=0.8,
random_state=params["random_seed"])
molecule_dataset_train = MoleculeDataset(X_train, y_train)
molecule_dataset_test = MoleculeDataset(X_test, y_val)
if params["use_gpu"] and torch.cuda.is_available():
device = torch.device('cuda:0')
gpu = 1
else:
device = torch.device('cpu')
gpu = 0
callbacks = [EarlyStopping(monitor='val_loss')]
if args.hyper:
params["molecule_dataset_train"] = molecule_dataset_train
params["molecule_dataset_test"] = molecule_dataset_test
params["device"] = device
params["gpu"] = gpu
params["es"] = True if args.es else False
results = ax.optimize(params["hyper"]["parameters"],
evaluation_function,
random_seed=params["random_seed"],
minimize=params["minimize"],
total_trials=params["hyper"]["trials"])
print(results)
else:
data_loader_train = data.DataLoader(molecule_dataset_train,
batch_size=params["batch_size"],
shuffle=False,
collate_fn=gcn_collate_fn)
data_loader_val = data.DataLoader(molecule_dataset_test,
batch_size=params["batch_size"],
shuffle=False,
collate_fn=gcn_collate_fn)
model = GraphConvModel(device_ext=device,
task=params["task"],
conv_layer_sizes=params["conv_layer_sizes"],
fingerprints_size=params["fingerprints_size"],
mlp_layer_sizes=params["mlp_layer_sizes"],
lr=params["lr"])
callbacks = []
checkpoint_callback = ModelCheckpoint(
monitor='val_' + params["metrics"],
dirpath=params["check_point"]["dirpath"],
filename="model-{epoch:02d}-{val_" + params["metrics"] + ":.5f}",
save_top_k=params["check_point"]["save_top_k"],
mode='min' if params["minimize"] else 'max')
callbacks.append(checkpoint_callback)
if args.es:
early_stop_callback = EarlyStopping(
min_delta=params["early_stopping"]["min_delta"],
patience=params["early_stopping"]["patience"],
verbose=params["early_stopping"]["verbose"],
monitor="val_" + params["metrics"],
mode='min' if params["minimize"] else 'max')
callbacks.append(early_stop_callback)
trainer = pl.Trainer(max_epochs=params["num_epochs"],
gpus=gpu,
callbacks=callbacks)
trainer.fit(model, data_loader_train, data_loader_val)
print_result(trainer)
def _opt_svaal(self, objective_name="val_f1_macro", minimise=None):
""" Optimisation routine for SVAAL model
Notes
-----
This model differs from SVAE and FDP as SVAAL is not necessarily trained on full data.
"""
start_time = datetime.now()
best_parameters, best_values, experiment, model = optimize(
parameters=[
{
"name": "epochs",
"type": "range",
"value_type": "int",
"bounds": [10, 100],
},
{
"name": "batch_size",
"type": "range",
"value_type": "int",
"bounds": [16, 64],
},
# {
# "name": "tl_embedding_dim",
# "type": "fixed",
# "value_type": "int",
# "value": 64,
# },
# {
# "name": "tl_hidden_dim",
# "type": "fixed",
# "value_type": "int",
# "value": 64,
# },
# {
# "name": "tl_rnn_type",
# "type": "fixed",
# "value_type": "str",
# "value": "gru",
# },
# {
# "name": "tl_learning_rate",
# "type": "fixed",
# "value_type": "float",
# "value": 0.01,
# },
{
"name": "latent_size",
"type": "range",
"value_type": "int",
"bounds": [16, 128],
},
{
"name": "disc_fc_dim",
"type": "range",
"value_type": "int",
"bounds": [64, 256]
},
{
"name": "disc_learning_rate",
"type": "range",
"value_type": "float",
"bounds": [0.0001, 0.1]
},
# {
# "name": "svae_rnn_type",
# "type": "fixed",
# "value_type": "str",
# "value": "gru",
# },
{
"name": "svae_embedding_dim",
"type": "range",
"value_type": "int",
"bounds": [128, 1024],
},
{
"name": "svae_hidden_dim",
"type": "range",
"value_type": "int",
"bounds": [128, 1024],
},
# {
# "name": "svae_num_layers",
# "type": "fixed",
# "value_type": "int",
# "value": 1,
# },
# {
# "name": "svae_bidirectional",
# "type": "fixed",
# "value_type": "bool",
# "value": False,
# },
{
"name": "svae_word_dropout",
"type": "range",
"value_type": "float",
"bounds": [0, 1],
},
{
"name": "svae_embedding_dropout",
"type": "range",
"value_type": "float",
"bounds": [0, 1],
},
{
"name": "svae_k",
"type": "range",
"value_type": "float",
"bounds": [0, 1],
},
{
"name": "svae_x0",
"type": "range",
"value_type": "int",
"bounds": [250, 5000],
},
# {
# "name": "svae_adv_hyperparameter",
# "type": "fixed",
# "value_type": "int",
# "value": 1,
# },
{
"name": "svae_learning_rate",
"type": "range",
"value_type": "float",
"bounds": [0.0001, 0.1]
},
],
evaluation_function=self.exp.train_single_cycle,
minimize=minimise,
objective_name=objective_name,
total_trials=self.trials)
finish_time = datetime.now()
run_time = (finish_time - start_time).total_seconds() / 60
data = {
"name": "SEQ-SVAE-BBN-GRU",
"info": {
"start timestamp": start_time,
"finish timestamp": finish_time,
"run time": run_time
},
"settings": {
"trials": self.trials,
"object name": objective_name,
"minimise": minimise
},
"results": {
"best parameters": best_parameters,
"best value": best_values[0][objective_name]
}
}
print(data)
# Post results to mongodb
self.mongo_coll_conn.post(data)
def _opt_full_data_performance(self,
objective_name="f1_macro",
minimise=None):
""" Optimisation routine for full data performance of task learner """
start_time = datetime.now()
best_parameters, best_values, _, _ = optimize(
parameters=[
{
"name": "epochs",
"type": "range",
"value_type": "int",
"bounds": [16, 128]
},
{
"name": "batch_size",
"type": "range",
"value_type": "int",
"bounds": [16, 64],
},
{
"name": "tl_rnn_type",
"type": "fixed",
"value_type": "str",
"value": "gru",
},
{
"name": "tl_embedding_dim",
"type": "range",
"value_type": "int",
"bounds": [128, 1024],
},
{
"name": "tl_hidden_dim",
"type": "range",
"value_type": "int",
"bounds": [128, 1024],
},
{
"name":
"learning_rate", # TODO: Will need to update to tl_ in the future
"type": "range",
"value_type": "float",
"bounds": [0.00001, 0.1]
},
],
evaluation_function=self.exp._full_data_performance,
minimize=minimise,
objective_name=objective_name,
total_trials=self.trials)
finish_time = datetime.now()
run_time = (finish_time - start_time).total_seconds() / 60
# TODO: Will put into a decorated in the future...
data = {
"name": f"FDP-{self.task_type}-{self.data_name}",
"info": {
"start timestamp": start_time,
"finish timestamp": finish_time,
"run time": run_time
},
"config": self.config,
"settings": {
"trials": self.trials,
"object name": objective_name,
"minimise": minimise
},
"results": {
"best parameters": best_parameters,
"best value": best_values[0][objective_name]
}
}
# # Post results to mongodb
self.mongo_coll_conn.post(data)
def main():
""" This program's entrypoint. """
# Parse command line arguments.
psr = argparse.ArgumentParser(
description="Hyper-parameter optimizer for train.py.")
psr, psr_verify = cl_args.add_training(psr)
psr.add_argument("--opt-trials",
default=DEFAULT_TLS_OPT,
help="The number of optimization trials to run.",
type=int)
psr.add_argument(
"--exhaustive",
action="store_true",
help=("Try all combinations of parameters. Incompatible with "
"parameters of type \"range\"."))
args = psr_verify(psr.parse_args())
tls_opt = args.opt_trials
tls_cnf = args.conf_trials
no_rand = args.no_rand
# Define the optimization parameters.
params = [
{
"name": "data_dir",
"type": "fixed",
"value": args.data_dir
},
{
"name": "model",
"type": "choice",
"values": ["SvmSklearn", "LrSklearn"]
},
{
"name": "conf_trials",
"type": "fixed",
"value": tls_cnf
},
{
"name": "max_attempts",
"type": "fixed",
"value": args.max_attempts
},
{
"name": "num_gpus",
"type": "fixed",
"value": 0
},
{
"name": "warmup_percent",
"type": "fixed",
"value": 10
},
{
"name": "num_sims",
"type": "fixed",
"value": args.num_sims
},
# {
# "name": "train_batch",
# "type": "fixed",
# "value": 10
# },
# {
# "name": "learning_rate",
# "type": "range",
# "bounds": [0.0001, 0.01]
# },
# {
# "name": "momentum",
# "type": "range",
# "bounds": [0.09, 0.99]
# },
{
"name": "kernel",
"type": "choice",
"values": ["linear", "poly", "rbf", "sigmoid"]
},
# {
# "name": "degree",
# "type": "range",
# "bounds": [0, 20]
# },
# Represent degree as a choice parameter so that it is
# compatible with exhaustive mode.
{
"name": "degree",
"type": "choice",
"values": list(range(0, 21))
},
{
"name": "penalty",
"type": "choice",
"values": ["l1", "l2"]
},
{
"name": "no_rand",
"type": "fixed",
"value": no_rand
},
{
"name": "timeout_s",
"type": "fixed",
"value": args.timeout_s
},
{
"name": "out_dir",
"type": "fixed",
"value": args.out_dir
}
]
# If we are using early stopping, then "epochs" is unnecessary but
# "val_patience" and "val_improvement_thresh" are also candidates for
# optimization.
if args.early_stop:
params.extend([{
"name": "early_stop",
"type": "fixed",
"value": True
}, {
"name": "val_patience",
"type": "range",
"bounds": [5, 20]
}, {
"name": "val_improvement_thresh",
"type": "range",
"bounds": [0.01, 0.1]
}])
else:
params.extend([
# {
# "name": "epochs",
# "type": "range",
# "bounds": [1, 100]
# },
])
tim_srt_s = time.time()
if args.exhaustive:
for param in params:
assert param["type"] != "range", \
f"Exhaustive mode does not support range parameters: {param}"
fixed = {
param["name"]: param["value"]
for param in params if param["type"] == "fixed"
}
to_vary = [[(param["name"], value) for value in param["values"]]
for param in params if param["type"] == "choice"]
print(
f"Varying these parameters, with {tls_cnf} sub-trials(s) for each "
f"configuration: {[pairs[0][0] for pairs in to_vary]}")
cnfs = [{
**fixed,
**dict(params)
} for params in itertools.product(*to_vary)]
print(f"Total trials: {len(cnfs) * tls_cnf}")
if defaults.SYNC:
res = [train.run_trials(cnf)[0] for cnf in cnfs]
else:
with multiprocessing.Pool() as pol:
res = pol.map(lambda cnf: train.run_trials(cnf)[0], cnfs)
best_idx = np.argmin(np.array(res))
best_params = cnfs[best_idx]
best_err = res[best_idx]
else:
print((f"Running {tls_opt} optimization trial(s), with {tls_cnf} "
"sub-trial(s) for each configuration."))
best_params, best_vals, _, _ = ax.optimize(
parameters=params,
evaluation_function=lambda cnf: train.run_trials(cnf)[0],
minimize=True,
total_trials=args.opt_trials,
random_seed=utils.SEED if no_rand else None)
best_err = best_vals[0]["objective"]
print((f"Done with hyper-parameter optimization - "
f"{time.time() - tim_srt_s:.2f} seconds"))
print(f"\nBest params: {best_params}")
print(f"Best error: {best_err:.4f}%")
def _opt_svae(self, objective_name="train_loss", minimise=None):
""" Optimisation routine for full data performance of SVAE """
start_time = datetime.now()
best_parameters, best_values, _, _ = optimize(
parameters=[
{
"name": "epochs",
"type": "range",
"value_type": "int",
"bounds": [25, 100],
},
{
"name": "batch_size",
"type": "range",
"value_type": "int",
"bounds": [16, 64],
},
{
"name": "svae_embedding_dim",
"type": "range",
"value_type": "int",
"bounds": [128, 1024],
},
{
"name": "svae_hidden_dim",
"type": "range",
"value_type": "int",
"bounds": [128, 1024],
},
{
"name": "svae_rnn_type",
"type": "fixed",
"value_type": "str",
"value": "gru",
},
{
"name": "svae_num_layers",
"type": "fixed",
"value_type": "int",
"value": 1,
},
{
"name": "svae_bidirectional",
"type": "fixed",
"value_type": "bool",
"value": True,
},
{
"name": "svae_latent_size",
"type": "range",
"value_type": "int",
"bounds": [16, 256],
},
{
"name": "svae_word_dropout",
"type": "range",
"value_type": "float",
"bounds": [0, 1],
},
{
"name": "svae_embedding_dropout",
"type": "range",
"value_type": "float",
"bounds": [0, 1],
},
{
"name": "svae_k",
"type": "range",
"value_type": "float",
"bounds": [0, 1],
},
{
"name": "svae_x0",
"type": "range",
"value_type": "int",
"bounds": [250, 5000],
},
],
evaluation_function=self.exp._svae,
minimize=minimise,
objective_name=objective_name,
total_trials=self.trials)
finish_time = datetime.now()
run_time = (finish_time - start_time).total_seconds() / 60
# TODO: Will put into a decorated in the future...
data = {
"name": "SVAE",
"info": {
"start timestamp": start_time,
"finish timestamp": finish_time,
"run time": run_time
},
"settings": {
"trials": self.trials,
"object name": objective_name,
"minimise": minimise
},
"results": {
"best parameters": best_parameters,
"best value": best_values[0][objective_name]
}
}
# # Post results to mongodb
self.mongo_coll_conn.post(data)
#Loss export stuff:
model_loss_dict = {}
best_parameters, values, experiment, model = ax.optimize(
parameters=[
{
"name": "fc1s",
"type": "range",
"bounds": [8, 128]
}, # [8, 64]
{
"name": "fc2s",
"type": "range",
"bounds": [4, 64]
}, # [4, 32]
{
"name": "fc3s",
"type": "range",
"bounds": [4, 64]
} # [4, 32]
],
evaluation_function=create_train_eval,
minimize=True, # minimize since objective is loss
total_trials=options.trials, # default 20
arms_per_trial=1, # default 1
objective_name='BCE_loss',
)
filename = 'BO_model_losses_{}b.json'.format(options.bits)
with open(os.path.join(options.outputDir, filename), 'w') as fp:
json.dump(model_loss_dict, fp)
best_parameters, best_values, _, _ = optimize(
# https://ax.dev/docs/core.html#search-space-and-parameters
parameters=[
{
"name": "INITIAL_LR",
"type": "range",
"bounds": [5e-6, 5e-3]
},
{
"name": "GAMMA",
"type": "range",
"bounds": [0.8, 0.999]
},
{
"name": "ACTION_REPEATS",
"type": "choice",
"values": [1, 2, 3, 4]
},
{
"name": "UPDATE_FREQUENCY",
"type": "choice",
"values": [4, 16, 64]
},
{
"name": "MLP_FEATURES_HIDDEN",
"type": "choice",
"values": ['(64, 64)', '(128, 128)', '(512, 512)', '(512, )']
},
{
"name": "OUTPUT_FC_HIDDEN_SIZES",
"type": "choice",
"values": ['(64, 64)', '(128, 64)', '(64, 32)', '(512, )']
},
{
"name": "OUTPUT_HIDDEN_DROPOUT",
"type": "choice",
"values": [0, 0.1, 0.2, 0.3, 0.4, 0.5],
"value_type": 'float',
},
{
"name": "CATEGORICAL",
"type": "choice",
"values": [True, False]
},
{
"name": "SUPPORT_RANGE",
"type": "choice",
"values": ['(-10, 10)', '(-50, 50)', '(-100, 100)']
},
{
"name": "NOISY",
"type": "choice",
"values": [True, False]
},
{
"name": "DUELING",
"type": "choice",
"values": [True, False]
},
],
evaluation_function=banana_tuning,
minimize=False,
arms_per_trial=1,
total_trials=NUM_TRIALS)
ret = evaluate(params, x_train, y_train, x_valid, y_valid)
return {"validation_loss": ret["validation_loss"]}
best_parameters, best_values, _, _ = optimize(
parameters=[{
"name": "batch_size",
"type": "range",
"bounds": [16, 512]
}, {
"name": "epochs",
"type": "range",
"bounds": [1, 400]
}, {
"name": "learning_rate",
"type": "range",
"bounds": [1e-6, 0.01],
"log_scale": True
}, {
"name": "hidden_dim",
"type": "range",
"bounds": [10, 200]
}, {
"name": "layer_dim",
"type": "range",
"bounds": [1, 8]
}],
evaluation_function=eval_with_data,
minimize=True,
objective_name="validation_loss")
print("Best parameters:", best_parameters)
print("Best values:", best_values)
# Setup dataloaders with our dataset
train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size,
shuffle=True, num_workers=10,
pin_memory=True) # FFS, have to use numworkers = 0 because apparently h5 objects can't be pickled, https://github.com/WuJie1010/Facial-Expression-Recognition.Pytorch/issues/69
validate_loader = torch.utils.data.DataLoader(val_dataset, batch_size=batch_size,
shuffle=True, num_workers=10, pin_memory=True)
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=test_size,
shuffle=False, num_workers=10, pin_memory=True)
best_parameters, values, experiment, model = ax.optimize(
parameters=[
{"name": "fc1s", "type": "range", "bounds": [8, 64]},
{"name": "fc2s", "type": "range", "bounds": [4, 32]},
{"name": "fc3s", "type": "range", "bounds": [4, 32]}
],
evaluation_function=create_train_eval,
objective_name='accuracy',
)
print("~*~*~*~*~*~*~*~RESULTS~*~*~*~*~*~*~*~")
print("Best Params: {}".format(best_parameters))
means, covariances = values
print("Means: {}".format(means))
print("Covariances: {}".format(covariances))
# Time for filenames
now = datetime.now()
time = now.strftime("%d-%m-%Y_%H-%M-%S")
results_file = open("{}BO_Results_Summary-{}b_{}".format(options.outputDir,options.bits,time),"a")
