def set_algorithm(experiment_name, config):
'''
Configure search algorithm.
'''
if args.algorithm == 'hyperopt':
algorithm = HyperOptSearch(points_to_evaluate=best_params)
elif args.algorithm == 'ax':
ax_client = AxClient(enforce_sequential_optimization=False)
ax_client.create_experiment(name=experiment_name,
parameters=config,
objective_name="minimum",
minimize=True)
algorithm = AxSearch(ax_client=ax_client,
points_to_evaluate=best_params)
elif args.algorithm == 'nevergrad':
algorithm = NevergradSearch(
points_to_evaluate=best_params,
optimizer=ng.optimizers.registry["PortfolioDiscreteOnePlusOne"])
elif args.algorithm == 'optuna':
algorithm = OptunaSearch(points_to_evaluate=best_params,
seed=args.seed)
elif args.algorithm == 'pbt':
algorithm = PopulationBasedTraining(
time_attr="training_iteration",
perturbation_interval=args.perturbation,
hyperparam_mutations=config,
synch=True)
elif args.algorithm == 'random':
algorithm = BasicVariantGenerator(max_concurrent=args.jobs)
if args.algorithm not in ['random', 'pbt']:
algorithm = ConcurrencyLimiter(algorithm, max_concurrent=args.jobs)
return algorithm
def testNevergrad(self):
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
searcher = NevergradSearch(space=self.config,
metric=self.metric_name,
mode="max",
optimizer=ng.optimizers.RandomSearch)
self._save(searcher)
searcher = NevergradSearch(space=self.config,
metric=self.metric_name,
mode="max",
optimizer=ng.optimizers.RandomSearch)
self._restore(searcher)
def backtest_tune(ticks: np.ndarray, backtest_config: dict, current_best: Union[dict, list] = None):
config = create_config(backtest_config)
n_days = round_((ticks[-1][2] - ticks[0][2]) / (1000 * 60 * 60 * 24), 0.1)
session_dirpath = make_get_filepath(os.path.join('reports', backtest_config['exchange'], backtest_config['symbol'],
f"{n_days}_days_{ts_to_date(time())[:19].replace(':', '')}", ''))
iters = 10
if 'iters' in backtest_config:
iters = backtest_config['iters']
else:
print('Parameter iters should be defined in the configuration. Defaulting to 10.')
num_cpus = 2
if 'num_cpus' in backtest_config:
num_cpus = backtest_config['num_cpus']
else:
print('Parameter num_cpus should be defined in the configuration. Defaulting to 2.')
n_particles = 10
if 'n_particles' in backtest_config:
n_particles = backtest_config['n_particles']
phi1 = 1.4962
phi2 = 1.4962
omega = 0.7298
if 'options' in backtest_config:
phi1 = backtest_config['options']['c1']
phi2 = backtest_config['options']['c2']
omega = backtest_config['options']['w']
current_best_params = []
if current_best:
if type(current_best) == list:
for c in current_best:
c = clean_start_config(c, config, backtest_config['ranges'])
current_best_params.append(c)
else:
current_best = clean_start_config(current_best, config, backtest_config['ranges'])
current_best_params.append(current_best)
ray.init(num_cpus=num_cpus, logging_level=logging.FATAL, log_to_driver=False)
pso = ng.optimizers.ConfiguredPSO(transform='identity', popsize=n_particles, omega=omega, phip=phi1, phig=phi2)
algo = NevergradSearch(optimizer=pso, points_to_evaluate=current_best_params)
algo = ConcurrencyLimiter(algo, max_concurrent=num_cpus)
scheduler = AsyncHyperBandScheduler()
analysis = tune.run(tune.with_parameters(backtest, ticks=ticks), metric='objective', mode='max', name='search',
search_alg=algo, scheduler=scheduler, num_samples=iters, config=config, verbose=1,
reuse_actors=True, local_dir=session_dirpath,
progress_reporter=LogReporter(metric_columns=['daily_gain', 'closest_liquidation', 'objective'],
parameter_columns=[k for k in backtest_config['ranges']]))
ray.shutdown()
df = analysis.results_df
df.reset_index(inplace=True)
df.drop(columns=['trial_id', 'time_this_iter_s', 'done', 'timesteps_total', 'episodes_total', 'training_iteration',
'experiment_id', 'date', 'timestamp', 'time_total_s', 'pid', 'hostname', 'node_ip',
'time_since_restore', 'timesteps_since_restore', 'iterations_since_restore', 'experiment_tag'],
inplace=True)
df.to_csv(os.path.join(backtest_config['session_dirpath'], 'results.csv'), index=False)
print('Best candidate found:')
pprint.pprint(analysis.best_config)
plot_wrap(backtest_config, ticks, clean_result_config(analysis.best_config))
return analysis
def testConvergenceNevergrad(self):
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
np.random.seed(0)
searcher = NevergradSearch(optimizer=ng.optimizers.PSO)
analysis = self._testConvergence(searcher, patience=50, top=5)
assert math.isclose(analysis.best_config["x"], 0, abs_tol=1e-3)
def backtest_tune(ticks: np.ndarray, backtest_config: dict, current_best: Union[dict, list] = None):
config = create_config(backtest_config)
n_days = round_((ticks[-1][2] - ticks[0][2]) / (1000 * 60 * 60 * 24), 0.1)
session_dirpath = make_get_filepath(os.path.join('reports', backtest_config['exchange'], backtest_config['symbol'],
f"{n_days}_days_{ts_to_date(time())[:19].replace(':', '')}", ''))
iters = 10
if 'iters' in backtest_config:
iters = backtest_config['iters']
else:
print('Parameter iters should be defined in the configuration. Defaulting to 10.')
num_cpus = 2
if 'num_cpus' in backtest_config:
num_cpus = backtest_config['num_cpus']
else:
print('Parameter num_cpus should be defined in the configuration. Defaulting to 2.')
n_particles = 10
if 'n_particles' in backtest_config:
n_particles = backtest_config['n_particles']
phi1 = 1.4962
phi2 = 1.4962
omega = 0.7298
if 'options' in backtest_config:
phi1 = backtest_config['options']['c1']
phi2 = backtest_config['options']['c2']
omega = backtest_config['options']['w']
current_best_params = []
if current_best:
if type(current_best) == list:
for c in current_best:
c = clean_start_config(c, config, backtest_config['ranges'])
current_best_params.append(c)
else:
current_best = clean_start_config(current_best, config, backtest_config['ranges'])
current_best_params.append(current_best)
ray.init(num_cpus=num_cpus, logging_level=logging.FATAL, log_to_driver=False)
pso = ng.optimizers.ConfiguredPSO(transform='identity', popsize=n_particles, omega=omega, phip=phi1, phig=phi2)
algo = NevergradSearch(optimizer=pso, points_to_evaluate=current_best_params)
algo = ConcurrencyLimiter(algo, max_concurrent=num_cpus)
scheduler = AsyncHyperBandScheduler()
analysis = tune.run(tune.with_parameters(wrap_backtest, ticks=ticks), metric='objective', mode='max', name='search',
search_alg=algo, scheduler=scheduler, num_samples=iters, config=config, verbose=1,
reuse_actors=True, local_dir=session_dirpath,
progress_reporter=LogReporter(metric_columns=['daily_gain',
'closest_liquidation',
'max_hours_between_fills',
'objective'],
parameter_columns=[k for k in backtest_config['ranges'] if type(
config[k]) == ray.tune.sample.Float or type(
config[k]) == ray.tune.sample.Integer]))
ray.shutdown()
return analysis
def set_basic_conf(self):
instrumentation = 2
parameter_names = ["height", "width"]
optimizer = optimizerlib.OnePlusOne(instrumentation)
def cost(space, reporter):
reporter(
mean_loss=(space["height"] - 14)**2 - abs(space["width"] - 3))
search_alg = NevergradSearch(
optimizer, parameter_names, metric="mean_loss", mode="min")
return search_alg, cost
def set_basic_conf(self):
instrumentation = 2
parameter_names = ["height", "width"]
optimizer = optimizerlib.OnePlusOne(instrumentation)
def cost(space, reporter):
reporter(loss=(space["height"] - 14)**2 - abs(space["width"] - 3))
search_alg = NevergradSearch(
optimizer,
parameter_names,
metric="loss",
mode="min",
max_concurrent=1000, # Here to avoid breaking back-compat.
)
return search_alg, cost
def testNevergrad(self):
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
np.random.seed(2020) # At least one nan, inf, -inf and float
out = tune.run(
_invalid_objective,
search_alg=NevergradSearch(optimizer=ng.optimizers.RandomSearch),
config=self.config,
mode="max",
num_samples=16,
reuse_actors=False)
best_trial = out.best_trial
self.assertLessEqual(best_trial.config["report"], 2.0)
def testConvertNevergrad(self):
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
config = {
"a": tune.sample.Categorical([2, 3, 4]).uniform(),
"b": {
"x": tune.sample.Integer(0, 5).quantized(2),
"y": 4,
"z": tune.sample.Float(1e-4, 1e-2).loguniform()
}
}
converted_config = NevergradSearch.convert_search_space(config)
nevergrad_config = ng.p.Dict(
a=ng.p.Choice([2, 3, 4]),
b=ng.p.Dict(x=ng.p.Scalar(lower=0, upper=5).set_integer_casting(),
z=ng.p.Log(lower=1e-4, upper=1e-2)))
searcher1 = NevergradSearch(optimizer=ng.optimizers.OnePlusOne,
space=converted_config)
searcher2 = NevergradSearch(optimizer=ng.optimizers.OnePlusOne,
space=nevergrad_config)
np.random.seed(1234)
config1 = searcher1.suggest("0")
np.random.seed(1234)
config2 = searcher2.suggest("0")
self.assertEqual(config1, config2)
self.assertIn(config1["a"], [2, 3, 4])
self.assertIn(config1["b"]["x"], list(range(5)))
self.assertLess(1e-4, config1["b"]["z"])
self.assertLess(config1["b"]["z"], 1e-2)
searcher = NevergradSearch(optimizer=ng.optimizers.OnePlusOne,
metric="a",
mode="max")
analysis = tune.run(_mock_objective,
config=config,
search_alg=searcher,
num_samples=1)
trial = analysis.trials[0]
assert trial.config["a"] in [2, 3, 4]
def testNevergradBestParams(self):
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
config = {
"metric": tune.sample.Categorical([1, 2, 3, 4]).uniform(),
"a": tune.sample.Categorical(["t1", "t2", "t3", "t4"]).uniform(),
"b": tune.sample.Integer(0, 5),
"c": tune.sample.Float(1e-4, 1e-1).loguniform()
}
best_params = [{
"metric": 1,
"a": "t1",
"b": 1,
"c": 1e-1
}, {
"metric": 2,
"a": "t2",
"b": 2,
"c": 1e-2
}]
searcher = NevergradSearch(optimizer=ng.optimizers.OnePlusOne,
points_to_evaluate=best_params)
analysis = tune.run(_mock_objective,
config=config,
metric="metric",
mode="max",
search_alg=searcher,
num_samples=5)
for i in range(len(best_params)):
trial_config = analysis.trials[i].config
trial_config_dict = {
"metric": trial_config["metric"],
"a": trial_config["a"],
"b": trial_config["b"],
"c": trial_config["c"]
}
self.assertDictEqual(trial_config_dict, best_params[i])
def cifar10_main(method='BlendSearch',
num_samples=10,
max_num_epochs=100,
gpus_per_trial=2):
data_dir = os.path.abspath("test/data")
load_data(data_dir) # Download data for all trials before starting the run
if method == 'BlendSearch':
from flaml import tune
else:
from ray import tune
if method in ['BlendSearch', 'BOHB', 'Optuna']:
config = {
"l1": tune.randint(2, 8),
"l2": tune.randint(2, 8),
"lr": tune.loguniform(1e-4, 1e-1),
"num_epochs": tune.qloguniform(1, max_num_epochs, q=1),
"batch_size": tune.randint(1, 4) #tune.choice([2, 4, 8, 16])
}
else:
config = {
"l1": tune.randint(2, 9),
"l2": tune.randint(2, 9),
"lr": tune.loguniform(1e-4, 1e-1),
"num_epochs": tune.qloguniform(1, max_num_epochs + 1, q=1),
"batch_size": tune.randint(1, 5) #tune.choice([2, 4, 8, 16])
}
import ray
time_budget_s = 3600
start_time = time.time()
if method == 'BlendSearch':
result = tune.run(ray.tune.with_parameters(train_cifar,
data_dir=data_dir),
init_config={
"l1": 2,
"l2": 2,
"num_epochs": 1,
"batch_size": 4,
},
metric="loss",
mode="min",
max_resource=max_num_epochs,
min_resource=1,
report_intermediate_result=True,
resources_per_trial={
"cpu": 2,
"gpu": gpus_per_trial
},
config=config,
local_dir='logs/',
num_samples=num_samples,
time_budget_s=time_budget_s,
use_ray=True)
else:
if 'ASHA' == method:
algo = None
elif 'BOHB' == method:
from ray.tune.schedulers import HyperBandForBOHB
from ray.tune.suggest.bohb import TuneBOHB
algo = TuneBOHB()
scheduler = HyperBandForBOHB(max_t=max_num_epochs)
elif 'Optuna' == method:
from ray.tune.suggest.optuna import OptunaSearch
algo = OptunaSearch()
elif 'CFO' == method:
from flaml import CFO
algo = CFO(points_to_evaluate=[{
"l1": 2,
"l2": 2,
"num_epochs": 1,
"batch_size": 4,
}])
elif 'Nevergrad' == method:
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne)
if method != 'BOHB':
from ray.tune.schedulers import ASHAScheduler
scheduler = ASHAScheduler(max_t=max_num_epochs, grace_period=1)
result = tune.run(tune.with_parameters(train_cifar, data_dir=data_dir),
resources_per_trial={
"cpu": 2,
"gpu": gpus_per_trial
},
config=config,
metric="loss",
mode="min",
num_samples=num_samples,
time_budget_s=time_budget_s,
scheduler=scheduler,
search_alg=algo)
ray.shutdown()
logger.info(f"method={method}")
logger.info(f"n_samples={num_samples}")
logger.info(f"time={time.time()-start_time}")
best_trial = result.get_best_trial("loss", "min", "all")
logger.info("Best trial config: {}".format(best_trial.config))
logger.info("Best trial final validation loss: {}".format(
best_trial.metric_analysis["loss"]["min"]))
logger.info("Best trial final validation accuracy: {}".format(
best_trial.metric_analysis["accuracy"]["max"]))
best_trained_model = Net(2**best_trial.config["l1"],
2**best_trial.config["l2"])
device = "cpu"
if torch.cuda.is_available():
device = "cuda:0"
if gpus_per_trial > 1:
best_trained_model = nn.DataParallel(best_trained_model)
best_trained_model.to(device)
checkpoint_path = os.path.join(best_trial.checkpoint.value, "checkpoint")
model_state, optimizer_state = torch.load(checkpoint_path)
best_trained_model.load_state_dict(model_state)
test_acc = _test_accuracy(best_trained_model, device)
logger.info("Best trial test set accuracy: {}".format(test_acc))
def testConvertNevergrad(self):
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
# Grid search not supported, should raise ValueError
with self.assertRaises(ValueError):
NevergradSearch.convert_search_space(
{"grid": tune.grid_search([0, 1])})
config = {
"a": tune.sample.Categorical([2, 3, 4]).uniform(),
"b": {
"x": tune.sample.Integer(0, 5).quantized(2),
"y": 4,
"z": tune.sample.Float(1e-4, 1e-2).loguniform()
}
}
converted_config = NevergradSearch.convert_search_space(config)
nevergrad_config = ng.p.Dict(
a=ng.p.Choice([2, 3, 4]),
b=ng.p.Dict(x=ng.p.Scalar(lower=0, upper=5).set_integer_casting(),
z=ng.p.Log(lower=1e-4, upper=1e-2)))
searcher1 = NevergradSearch(optimizer=ng.optimizers.OnePlusOne,
space=converted_config,
metric="a",
mode="max")
searcher2 = NevergradSearch(optimizer=ng.optimizers.OnePlusOne,
space=nevergrad_config,
metric="a",
mode="max")
np.random.seed(1234)
config1 = searcher1.suggest("0")
np.random.seed(1234)
config2 = searcher2.suggest("0")
self.assertEqual(config1, config2)
self.assertIn(config1["a"], [2, 3, 4])
self.assertIn(config1["b"]["x"], list(range(5)))
self.assertLess(1e-4, config1["b"]["z"])
self.assertLess(config1["b"]["z"], 1e-2)
searcher = NevergradSearch(optimizer=ng.optimizers.OnePlusOne,
metric="a",
mode="max")
analysis = tune.run(_mock_objective,
config=config,
search_alg=searcher,
num_samples=1)
trial = analysis.trials[0]
assert trial.config["a"] in [2, 3, 4]
mixed_config = {
"a": tune.uniform(5, 6),
"b": tune.uniform(8, 9) # Cannot mix Nevergrad cfg and tune
}
searcher = NevergradSearch(space=mixed_config,
optimizer=ng.optimizers.OnePlusOne,
metric="a",
mode="max")
config = searcher.suggest("0")
self.assertTrue(5 <= config["a"] <= 6)
self.assertTrue(8 <= config["b"] <= 9)
def hparams(algorithm, scheduler, num_samples, tensorboard, bare):
from glob import glob
import tensorflow.summary
from tensorflow import random as tfrandom, int64 as tfint64
from ray import init as init_ray, shutdown as shutdown_ray
from ray import tune
from wandb.ray import WandbLogger
from wandb import sweep as wandbsweep
from wandb.apis import CommError as wandbCommError
# less summaries are logged if MLENCRYPT_TB is TRUE (for efficiency)
# TODO: use tf.summary.record_if?
environ["MLENCRYPT_TB"] = str(tensorboard).upper()
environ["MLENCRYPT_BARE"] = str(bare).upper()
if getenv('MLENCRYPT_TB', 'FALSE') == 'TRUE' and \
getenv('MLENCRYPT_BARE', 'FALSE') == 'TRUE':
raise ValueError('TensorBoard logging cannot be enabled in bare mode.')
logdir = f'logs/hparams/{datetime.now()}'
# "These results show that K = 3 is the optimal choice for the
# cryptographic application of neural synchronization. K = 1 and K = 2 are
# too insecure in regard to the geometric attack. And for K > 3 the effort
# of A and B grows exponentially with increasing L, while the simple attack
# is quite successful in the limit K -> infinity. Consequently, one should
# only use Tree Parity Machines with three hidden units for the neural
# key-exchange protocol." (Ruttor, 2006)
# https://arxiv.org/pdf/0711.2411.pdf#page=59
update_rules = [
'random-same',
# 'random-different-A-B-E', 'random-different-A-B',
'hebbian',
'anti_hebbian',
'random_walk'
]
K_bounds = {'min': 4, 'max': 8}
N_bounds = {'min': 4, 'max': 8}
L_bounds = {'min': 4, 'max': 8}
# TODO: don't use *_bounds.values() since .values doesn't preserve order
def get_session_num(logdir):
current_runs = glob(join(logdir, "run-*"))
if current_runs:
last_run_path = current_runs[-1]
last_run_session_num = int(last_run_path.split('-')[-1])
return last_run_session_num + 1
else: # there are no runs yet, start at 0
return 0
def trainable(config, reporter):
"""
Args:
config (dict): Parameters provided from the search algorithm
or variant generation.
"""
if not isinstance(config['update_rule'], str):
update_rule = update_rules[int(config['update_rule'])]
else:
update_rule = config['update_rule']
K, N, L = int(config['K']), int(config['N']), int(config['L'])
run_name = f"run-{get_session_num(logdir)}"
run_logdir = join(logdir, run_name)
# for each attack, the TPMs should start with the same weights
initial_weights_tensors = get_initial_weights(K, N, L)
training_steps_ls = {}
eve_scores_ls = {}
losses_ls = {}
# for each attack, the TPMs should use the same inputs
seed = tfrandom.uniform([],
minval=0,
maxval=tfint64.max,
dtype=tfint64).numpy()
for attack in ['none', 'geometric']:
initial_weights = {
tpm: weights_tensor_to_variable(weights, tpm)
for tpm, weights in initial_weights_tensors.items()
}
tfrandom.set_seed(seed)
if tensorboard:
attack_logdir = join(run_logdir, attack)
attack_writer = tensorflow.summary.create_file_writer(
attack_logdir)
with attack_writer.as_default():
training_steps, sync_scores, loss = run(
update_rule, K, N, L, attack, initial_weights)
else:
training_steps, sync_scores, loss = run(
update_rule, K, N, L, attack, initial_weights)
training_steps_ls[attack] = training_steps
eve_scores_ls[attack] = sync_scores
losses_ls[attack] = loss
avg_training_steps = tensorflow.math.reduce_mean(
list(training_steps_ls.values()))
avg_eve_score = tensorflow.math.reduce_mean(
list(eve_scores_ls.values()))
mean_loss = tensorflow.math.reduce_mean(list(losses_ls.values()))
reporter(
avg_training_steps=avg_training_steps.numpy(),
avg_eve_score=avg_eve_score.numpy(),
mean_loss=mean_loss.numpy(),
done=True,
)
if algorithm == 'hyperopt':
from hyperopt import hp as hyperopt
from hyperopt.pyll.base import scope
from ray.tune.suggest.hyperopt import HyperOptSearch
space = {
'update_rule': hyperopt.choice(
'update_rule',
update_rules,
),
'K': scope.int(hyperopt.quniform('K', *K_bounds.values(), q=1)),
'N': scope.int(hyperopt.quniform('N', *N_bounds.values(), q=1)),
'L': scope.int(hyperopt.quniform('L', *L_bounds.values(), q=1)),
}
algo = HyperOptSearch(
space,
metric='mean_loss',
mode='min',
points_to_evaluate=[
{
'update_rule': 0,
'K': 3,
'N': 16,
'L': 8
},
{
'update_rule': 0,
'K': 8,
'N': 16,
'L': 8
},
{
'update_rule': 0,
'K': 8,
'N': 16,
'L': 128
},
],
)
elif algorithm == 'bayesopt':
from ray.tune.suggest.bayesopt import BayesOptSearch
space = {
'update_rule': (0, len(update_rules)),
'K': tuple(K_bounds.values()),
'N': tuple(N_bounds.values()),
'L': tuple(L_bounds.values()),
}
algo = BayesOptSearch(
space,
metric="mean_loss",
mode="min",
# TODO: what is utility_kwargs for and why is it needed?
utility_kwargs={
"kind": "ucb",
"kappa": 2.5,
"xi": 0.0
})
elif algorithm == 'nevergrad':
from ray.tune.suggest.nevergrad import NevergradSearch
from nevergrad import optimizers
from nevergrad import p as ngp
algo = NevergradSearch(
optimizers.TwoPointsDE(
ngp.Instrumentation(
update_rule=ngp.Choice(update_rules),
K=ngp.Scalar(lower=K_bounds['min'],
upper=K_bounds['max']).set_integer_casting(),
N=ngp.Scalar(lower=N_bounds['min'],
upper=N_bounds['max']).set_integer_casting(),
L=ngp.Scalar(lower=L_bounds['min'],
upper=L_bounds['max']).set_integer_casting(),
)),
None, # since the optimizer is already instrumented with kwargs
metric="mean_loss",
mode="min")
elif algorithm == 'skopt':
from skopt import Optimizer
from ray.tune.suggest.skopt import SkOptSearch
optimizer = Optimizer([
update_rules,
tuple(K_bounds.values()),
tuple(N_bounds.values()),
tuple(L_bounds.values())
])
algo = SkOptSearch(
optimizer,
["update_rule", "K", "N", "L"],
metric="mean_loss",
mode="min",
points_to_evaluate=[
['random-same', 3, 16, 8],
['random-same', 8, 16, 8],
['random-same', 8, 16, 128],
],
)
elif algorithm == 'dragonfly':
# TODO: doesn't work
from ray.tune.suggest.dragonfly import DragonflySearch
from dragonfly.exd.experiment_caller import EuclideanFunctionCaller
from dragonfly.opt.gp_bandit import EuclideanGPBandit
# from dragonfly.exd.experiment_caller import CPFunctionCaller
# from dragonfly.opt.gp_bandit import CPGPBandit
from dragonfly import load_config
domain_config = load_config({
"domain": [
{
"name": "update_rule",
"type": "discrete",
"dim": 1,
"items": update_rules
},
{
"name": "K",
"type": "int",
"min": K_bounds['min'],
"max": K_bounds['max'],
# "dim": 1
},
{
"name": "N",
"type": "int",
"min": N_bounds['min'],
"max": N_bounds['max'],
# "dim": 1
},
{
"name": "L",
"type": "int",
"min": L_bounds['min'],
"max": L_bounds['max'],
# "dim": 1
}
]
})
func_caller = EuclideanFunctionCaller(
None, domain_config.domain.list_of_domains[0])
optimizer = EuclideanGPBandit(func_caller, ask_tell_mode=True)
algo = DragonflySearch(
optimizer,
metric="mean_loss",
mode="min",
points_to_evaluate=[
['random-same', 3, 16, 8],
['random-same', 8, 16, 8],
['random-same', 8, 16, 128],
],
)
elif algorithm == 'bohb':
from ConfigSpace import ConfigurationSpace
from ConfigSpace import hyperparameters as CSH
from ray.tune.suggest.bohb import TuneBOHB
config_space = ConfigurationSpace()
config_space.add_hyperparameter(
CSH.CategoricalHyperparameter("update_rule", choices=update_rules))
config_space.add_hyperparameter(
CSH.UniformIntegerHyperparameter(name='K',
lower=K_bounds['min'],
upper=K_bounds['max']))
config_space.add_hyperparameter(
CSH.UniformIntegerHyperparameter(name='N',
lower=N_bounds['min'],
upper=N_bounds['max']))
config_space.add_hyperparameter(
CSH.UniformIntegerHyperparameter(name='L',
lower=L_bounds['min'],
upper=L_bounds['max']))
algo = TuneBOHB(config_space, metric="mean_loss", mode="min")
elif algorithm == 'zoopt':
from ray.tune.suggest.zoopt import ZOOptSearch
from zoopt import ValueType
space = {
"update_rule":
(ValueType.DISCRETE, range(0, len(update_rules)), False),
"K": (ValueType.DISCRETE,
range(K_bounds['min'], K_bounds['max'] + 1), True),
"N": (ValueType.DISCRETE,
range(N_bounds['min'], N_bounds['max'] + 1), True),
"L": (ValueType.DISCRETE,
range(L_bounds['min'], L_bounds['max'] + 1), True),
}
# TODO: change budget to a large value
algo = ZOOptSearch(budget=10,
dim_dict=space,
metric="mean_loss",
mode="min")
# TODO: use more appropriate arguments for schedulers:
# https://docs.ray.io/en/master/tune/api_docs/schedulers.html
if scheduler == 'fifo':
sched = None # Tune defaults to FIFO
elif scheduler == 'pbt':
from ray.tune.schedulers import PopulationBasedTraining
from random import randint
sched = PopulationBasedTraining(
metric="mean_loss",
mode="min",
hyperparam_mutations={
"update_rule": update_rules,
"K": lambda: randint(K_bounds['min'], K_bounds['max']),
"N": lambda: randint(N_bounds['min'], N_bounds['max']),
"L": lambda: randint(L_bounds['min'], L_bounds['max']),
})
elif scheduler == 'ahb' or scheduler == 'asha':
# https://docs.ray.io/en/latest/tune/api_docs/schedulers.html#asha-tune-schedulers-ashascheduler
from ray.tune.schedulers import AsyncHyperBandScheduler
sched = AsyncHyperBandScheduler(metric="mean_loss", mode="min")
elif scheduler == 'hb':
from ray.tune.schedulers import HyperBandScheduler
sched = HyperBandScheduler(metric="mean_loss", mode="min")
elif algorithm == 'bohb' or scheduler == 'bohb':
from ray.tune.schedulers import HyperBandForBOHB
sched = HyperBandForBOHB(metric="mean_loss", mode="min")
elif scheduler == 'msr':
from ray.tune.schedulers import MedianStoppingRule
sched = MedianStoppingRule(metric="mean_loss", mode="min")
init_ray(
address=getenv("ip_head"),
redis_password=getenv('redis_password'),
)
analysis = tune.run(
trainable,
name='mlencrypt_research',
config={
"monitor": True,
"env_config": {
"wandb": {
"project": "mlencrypt-research",
"sync_tensorboard": True,
},
},
},
# resources_per_trial={"cpu": 1, "gpu": 3},
local_dir='./ray_results',
export_formats=['csv'], # TODO: add other formats?
num_samples=num_samples,
loggers=[
tune.logger.JsonLogger, tune.logger.CSVLogger,
tune.logger.TBXLogger, WandbLogger
],
search_alg=algo,
scheduler=sched,
queue_trials=True,
)
try:
wandbsweep(analysis)
except wandbCommError:
# see https://docs.wandb.com/sweeps/ray-tune#feature-compatibility
pass
best_config = analysis.get_best_config(metric='mean_loss', mode='min')
print(f"Best config: {best_config}")
shutdown_ray()
def _test_xgboost(method="BlendSearch"):
try:
import ray
except ImportError:
return
if method == "BlendSearch":
from flaml import tune
else:
from ray import tune
search_space = {
"max_depth":
tune.randint(1, 9)
if method in ["BlendSearch", "BOHB", "Optuna"] else tune.randint(1, 9),
"min_child_weight":
tune.choice([1, 2, 3]),
"subsample":
tune.uniform(0.5, 1.0),
"eta":
tune.loguniform(1e-4, 1e-1),
}
max_iter = 10
for num_samples in [128]:
time_budget_s = 60
for n_cpu in [2]:
start_time = time.time()
# ray.init(address='auto')
if method == "BlendSearch":
analysis = tune.run(
train_breast_cancer,
config=search_space,
low_cost_partial_config={
"max_depth": 1,
},
cat_hp_cost={
"min_child_weight": [6, 3, 2],
},
metric="eval-logloss",
mode="min",
max_resource=max_iter,
min_resource=1,
scheduler="asha",
# You can add "gpu": 0.1 to allocate GPUs
resources_per_trial={"cpu": 1},
local_dir="logs/",
num_samples=num_samples * n_cpu,
time_budget_s=time_budget_s,
use_ray=True,
)
else:
if "ASHA" == method:
algo = None
elif "BOHB" == method:
from ray.tune.schedulers import HyperBandForBOHB
from ray.tune.suggest.bohb import TuneBOHB
algo = TuneBOHB(max_concurrent=n_cpu)
scheduler = HyperBandForBOHB(max_t=max_iter)
elif "Optuna" == method:
from ray.tune.suggest.optuna import OptunaSearch
algo = OptunaSearch()
elif "CFO" == method:
from flaml import CFO
algo = CFO(
low_cost_partial_config={
"max_depth": 1,
},
cat_hp_cost={
"min_child_weight": [6, 3, 2],
},
)
elif "CFOCat" == method:
from flaml.searcher.cfo_cat import CFOCat
algo = CFOCat(
low_cost_partial_config={
"max_depth": 1,
},
cat_hp_cost={
"min_child_weight": [6, 3, 2],
},
)
elif "Dragonfly" == method:
from ray.tune.suggest.dragonfly import DragonflySearch
algo = DragonflySearch()
elif "SkOpt" == method:
from ray.tune.suggest.skopt import SkOptSearch
algo = SkOptSearch()
elif "Nevergrad" == method:
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne)
elif "ZOOpt" == method:
from ray.tune.suggest.zoopt import ZOOptSearch
algo = ZOOptSearch(budget=num_samples * n_cpu)
elif "Ax" == method:
from ray.tune.suggest.ax import AxSearch
algo = AxSearch()
elif "HyperOpt" == method:
from ray.tune.suggest.hyperopt import HyperOptSearch
algo = HyperOptSearch()
scheduler = None
if method != "BOHB":
from ray.tune.schedulers import ASHAScheduler
scheduler = ASHAScheduler(max_t=max_iter, grace_period=1)
analysis = tune.run(
train_breast_cancer,
metric="eval-logloss",
mode="min",
# You can add "gpu": 0.1 to allocate GPUs
resources_per_trial={"cpu": 1},
config=search_space,
local_dir="logs/",
num_samples=num_samples * n_cpu,
time_budget_s=time_budget_s,
scheduler=scheduler,
search_alg=algo,
)
# # Load the best model checkpoint
# import os
# best_bst = xgb.Booster()
# best_bst.load_model(os.path.join(analysis.best_checkpoint,
# "model.xgb"))
best_trial = analysis.get_best_trial("eval-logloss", "min", "all")
accuracy = 1.0 - best_trial.metric_analysis["eval-error"]["min"]
logloss = best_trial.metric_analysis["eval-logloss"]["min"]
logger.info(f"method={method}")
logger.info(f"n_samples={num_samples*n_cpu}")
logger.info(f"time={time.time()-start_time}")
logger.info(f"Best model eval loss: {logloss:.4f}")
logger.info(f"Best model total accuracy: {accuracy:.4f}")
logger.info(f"Best model parameters: {best_trial.config}")
def get_raytune_search_alg(raytune_cfg, seeds=False):
if (raytune_cfg["sched"] == "pbt") or (raytune_cfg["sched"] == "pb2"):
if raytune_cfg["search_alg"] is not None:
print(
"INFO: Using schedule '{}' is not compatible with Ray Tune search algorithms."
.format(raytune_cfg["sched"]))
print(
"INFO: Uing the Ray Tune {} scheduler without search algorithm"
.format(raytune_cfg["sched"]))
return None
if (raytune_cfg["sched"] == "bohb") or (raytune_cfg["sched"] == "BOHB"):
print(
"INFO: Using TuneBOHB search algorithm since it is required for BOHB shedule"
)
if seeds:
seed = 1234
else:
seed = None
return TuneBOHB(metric=raytune_cfg["default_metric"],
mode=raytune_cfg["default_mode"],
seed=seed)
# requires pip install bayesian-optimization
if raytune_cfg["search_alg"] == "bayes":
print("INFO: Using BayesOptSearch")
return BayesOptSearch(
metric=raytune_cfg["default_metric"],
mode=raytune_cfg["default_mode"],
random_search_steps=raytune_cfg["bayes"]["n_random_steps"],
)
# requires pip install hyperopt
if raytune_cfg["search_alg"] == "hyperopt":
print("INFO: Using HyperOptSearch")
return HyperOptSearch(
metric=raytune_cfg["default_metric"],
mode=raytune_cfg["default_mode"],
n_initial_points=raytune_cfg["hyperopt"]["n_random_steps"],
# points_to_evaluate=,
)
if raytune_cfg["search_alg"] == "scikit":
print("INFO: Using bayesian optimization from scikit-learn")
return SkOptSearch(
metric=raytune_cfg["default_metric"],
mode=raytune_cfg["default_mode"],
convert_to_python=True,
)
if raytune_cfg["search_alg"] == "nevergrad":
print("INFO: Using bayesian optimization from nevergrad")
return NevergradSearch(
optimizer=ng.optimizers.BayesOptim(
pca=False,
init_budget=raytune_cfg["nevergrad"]["n_random_steps"]),
metric=raytune_cfg["default_metric"],
mode=raytune_cfg["default_mode"],
)
# HEBO is not yet supported
# if (raytune_cfg["search_alg"] == "hebo") or (raytune_cfg["search_alg"] == "HEBO"):
# print("Using HEBOSearch")
# return HEBOSearch(
# metric=raytune_cfg["default_metric"],
# mode=raytune_cfg["default_mode"],
# # max_concurrent=8,
# )
else:
print("INFO: Not using any Ray Tune search algorithm")
return None
params = {}
params_bool = {}
SELECT_LAMBDA_RANGE = [0.02, 0.03, 0.04, 0.05]
SELECT_PRIOR_RANGE = [
0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.11, 0.12, 0.13, 0.14, 0.15
]
SELECT_CONT_RANGE = [0.04, 0.06, 0.08, 0.10]
if not os.path.exists('out'):
os.makedirs('out')
name = "randomMTM1"
algo = NevergradSearch(optimizer=ng.optimizers.RandomSearch,
metric="score",
mode="max")
algo = ConcurrencyLimiter(algo, max_concurrent=1)
scheduler = ASHAScheduler()
from ray import tune
analysis = tune.run(
train_MTM,
name=name,
scheduler=scheduler,
reuse_actors=False,
search_alg=algo,
verbose=2,
checkpoint_at_end=False,
num_samples=32, # 64
# export_formats=[ExportFormat.MODEL],
def backtest_tune(ticks: np.ndarray,
backtest_config: dict,
current_best: Union[dict, list] = None):
config = create_config(backtest_config)
n_days = round_((ticks[-1][2] - ticks[0][2]) / (1000 * 60 * 60 * 24), 0.1)
backtest_config['optimize_dirpath'] = os.path.join(
backtest_config['optimize_dirpath'],
ts_to_date(time())[:19].replace(':', ''), '')
if 'iters' in backtest_config:
iters = backtest_config['iters']
else:
print(
'Parameter iters should be defined in the configuration. Defaulting to 10.'
)
iters = 10
if 'num_cpus' in backtest_config:
num_cpus = backtest_config['num_cpus']
else:
print(
'Parameter num_cpus should be defined in the configuration. Defaulting to 2.'
)
num_cpus = 2
n_particles = backtest_config[
'n_particles'] if 'n_particles' in backtest_config else 10
phi1 = 1.4962
phi2 = 1.4962
omega = 0.7298
if 'options' in backtest_config:
phi1 = backtest_config['options']['c1']
phi2 = backtest_config['options']['c2']
omega = backtest_config['options']['w']
current_best_params = []
if current_best:
if type(current_best) == list:
for c in current_best:
c = clean_start_config(c, config, backtest_config['ranges'])
if c not in current_best_params:
current_best_params.append(c)
else:
current_best = clean_start_config(current_best, config,
backtest_config['ranges'])
current_best_params.append(current_best)
ray.init(num_cpus=num_cpus,
logging_level=logging.FATAL,
log_to_driver=False)
pso = ng.optimizers.ConfiguredPSO(transform='identity',
popsize=n_particles,
omega=omega,
phip=phi1,
phig=phi2)
algo = NevergradSearch(optimizer=pso,
points_to_evaluate=current_best_params)
algo = ConcurrencyLimiter(algo, max_concurrent=num_cpus)
scheduler = AsyncHyperBandScheduler()
if 'wfo' in config and config['wfo']:
print('\n\nwalk forward optimization\n\n')
wfo = WFO(ticks, backtest_config, P_train=0.5).set_train_N(4)
backtest_wrap = lambda config: tune_report(wfo.backtest(config))
else:
print('\n\nsimple sliding window optimization\n\n')
backtest_wrap = tune.with_parameters(simple_sliding_window_wrap,
ticks=ticks)
analysis = tune.run(backtest_wrap,
metric='objective',
mode='max',
name='search',
search_alg=algo,
scheduler=scheduler,
num_samples=iters,
config=config,
verbose=1,
reuse_actors=True,
local_dir=backtest_config['optimize_dirpath'],
progress_reporter=LogReporter(
metric_columns=[
'daily_gain', 'closest_liquidation',
'max_hrs_no_fills',
'max_hrs_no_fills_same_side', 'objective'
],
parameter_columns=[
k for k in backtest_config['ranges']
if type(config[k]) == ray.tune.sample.Float
or type(config[k]) == ray.tune.sample.Integer
]),
raise_on_failed_trial=False)
ray.shutdown()
return analysis
def cifar10_main(method="BlendSearch",
num_samples=10,
max_num_epochs=100,
gpus_per_trial=1):
data_dir = os.path.abspath("test/data")
load_data(data_dir) # Download data for all trials before starting the run
if method == "BlendSearch":
from flaml import tune
else:
from ray import tune
if method in ["BOHB"]:
config = {
"l1": tune.randint(2, 8),
"l2": tune.randint(2, 8),
"lr": tune.loguniform(1e-4, 1e-1),
"num_epochs": tune.qloguniform(1, max_num_epochs, q=1),
"batch_size": tune.randint(1, 4),
}
else:
config = {
"l1": tune.randint(2, 9),
"l2": tune.randint(2, 9),
"lr": tune.loguniform(1e-4, 1e-1),
"num_epochs": tune.loguniform(1, max_num_epochs),
"batch_size": tune.randint(1, 5),
}
import ray
time_budget_s = 600
np.random.seed(7654321)
start_time = time.time()
if method == "BlendSearch":
result = tune.run(
ray.tune.with_parameters(train_cifar, data_dir=data_dir),
config=config,
metric="loss",
mode="min",
low_cost_partial_config={"num_epochs": 1},
max_resource=max_num_epochs,
min_resource=1,
scheduler="asha",
resources_per_trial={
"cpu": 1,
"gpu": gpus_per_trial
},
local_dir="logs/",
num_samples=num_samples,
time_budget_s=time_budget_s,
use_ray=True,
)
else:
if "ASHA" == method:
algo = None
elif "BOHB" == method:
from ray.tune.schedulers import HyperBandForBOHB
from ray.tune.suggest.bohb import TuneBOHB
algo = TuneBOHB()
scheduler = HyperBandForBOHB(max_t=max_num_epochs)
elif "Optuna" == method:
from ray.tune.suggest.optuna import OptunaSearch
algo = OptunaSearch(seed=10)
elif "CFO" == method:
from flaml import CFO
algo = CFO(low_cost_partial_config={
"num_epochs": 1,
})
elif "Nevergrad" == method:
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne)
if method != "BOHB":
from ray.tune.schedulers import ASHAScheduler
scheduler = ASHAScheduler(max_t=max_num_epochs, grace_period=1)
result = tune.run(
tune.with_parameters(train_cifar, data_dir=data_dir),
resources_per_trial={
"cpu": 1,
"gpu": gpus_per_trial
},
config=config,
metric="loss",
mode="min",
num_samples=num_samples,
time_budget_s=time_budget_s,
scheduler=scheduler,
search_alg=algo,
)
ray.shutdown()
logger.info(f"method={method}")
logger.info(f"#trials={len(result.trials)}")
logger.info(f"time={time.time()-start_time}")
best_trial = result.get_best_trial("loss", "min", "all")
logger.info("Best trial config: {}".format(best_trial.config))
logger.info("Best trial final validation loss: {}".format(
best_trial.metric_analysis["loss"]["min"]))
logger.info("Best trial final validation accuracy: {}".format(
best_trial.metric_analysis["accuracy"]["max"]))
best_trained_model = Net(2**best_trial.config["l1"],
2**best_trial.config["l2"])
device = "cpu"
if torch.cuda.is_available():
device = "cuda:0"
if gpus_per_trial > 1:
best_trained_model = nn.DataParallel(best_trained_model)
best_trained_model.to(device)
checkpoint_path = os.path.join(best_trial.checkpoint.value, "checkpoint")
model_state, optimizer_state = torch.load(checkpoint_path)
best_trained_model.load_state_dict(model_state)
test_acc = _test_accuracy(best_trained_model, device)
logger.info("Best trial test set accuracy: {}".format(test_acc))
tune_kwargs = {
"num_samples": 10 if args.smoke_test else 50,
"config": {
"steps": 100,
"width": tune.uniform(0, 20),
"height": tune.uniform(-100, 100),
"activation": tune.choice(["relu", "tanh"])
}
}
# Optional: Pass the parameter space yourself
# space = ng.p.Dict(
# width=ng.p.Scalar(lower=0, upper=20),
# height=ng.p.Scalar(lower=-100, upper=100),
# activation=ng.p.Choice(choices=["relu", "tanh"])
# )
algo = NevergradSearch(
optimizer=ng.optimizers.OnePlusOne,
# space=space, # If you want to set the space manually
metric="mean_loss",
mode="min")
scheduler = AsyncHyperBandScheduler(metric="mean_loss", mode="min")
tune.run(easy_objective,
name="nevergrad",
search_alg=algo,
scheduler=scheduler,
**tune_kwargs)
ray.init()
config = {
"num_samples": 10 if args.smoke_test else 50,
"config": {
"iterations": 100,
},
"stop": {
"timesteps_total": 100
}
}
instrumentation = 2
parameter_names = ["height", "width"]
# With nevergrad v0.2.0+ the following is also possible:
# from nevergrad import instrumentation as inst
# instrumentation = inst.Instrumentation(
# height=inst.var.Array(1).bounded(0, 200).asfloat(),
# width=inst.var.OrderedDiscrete([0, 10, 20, 30, 40, 50]))
# parameter_names = None # names are provided by the instrumentation
optimizer = optimizerlib.OnePlusOne(instrumentation)
algo = NevergradSearch(optimizer,
parameter_names,
max_concurrent=4,
reward_attr="neg_mean_loss")
scheduler = AsyncHyperBandScheduler(reward_attr="neg_mean_loss")
run(easy_objective,
name="nevergrad",
search_alg=algo,
scheduler=scheduler,
**config)
config = {
"num_samples": 10 if args.smoke_test else 50,
"config": {
"iterations": 100,
},
"stop": {
"timesteps_total": 100
}
}
instrumentation = 2
parameter_names = ["height", "width"]
# With nevergrad v0.2.0+ the following is also possible:
# from nevergrad import instrumentation as inst
# instrumentation = inst.Instrumentation(
# height=inst.var.Array(1).bounded(0, 200).asfloat(),
# width=inst.var.OrderedDiscrete([0, 10, 20, 30, 40, 50]))
# parameter_names = None # names are provided by the instrumentation
optimizer = optimizerlib.OnePlusOne(instrumentation)
algo = NevergradSearch(optimizer,
parameter_names,
max_concurrent=4,
metric="mean_loss",
mode="min")
scheduler = AsyncHyperBandScheduler(metric="mean_loss", mode="min")
run(easy_objective,
name="nevergrad",
search_alg=algo,
scheduler=scheduler,
**config)
if __name__ == "__main__":
import argparse
from nevergrad.optimization import optimizerlib
parser = argparse.ArgumentParser()
parser.add_argument("--smoke-test",
action="store_true",
help="Finish quickly for testing")
args, _ = parser.parse_known_args()
ray.init()
config = {
"num_samples": 10 if args.smoke_test else 50,
"config": {
"iterations": 100,
},
"stop": {
"timesteps_total": 100
}
}
optimizer = optimizerlib.OnePlusOne(dimension=2)
algo = NevergradSearch(optimizer, ["height", "width"],
max_concurrent=4,
reward_attr="neg_mean_loss")
scheduler = AsyncHyperBandScheduler(reward_attr="neg_mean_loss")
run(easy_objective,
name="nevergrad",
search_alg=algo,
scheduler=scheduler,
**config)
def backtest_tune(data: np.ndarray,
config: dict,
current_best: Union[dict, list] = None):
memory = int(sys.getsizeof(data) * 1.2)
virtual_memory = psutil.virtual_memory()
print(f'data size in mb {memory / (1000 * 1000):.4f}')
if (virtual_memory.available - memory) / virtual_memory.total < 0.1:
print(
"Available memory would drop below 10%. Please reduce the time span."
)
return None
config = create_config(config)
print('tuning:')
for k, v in config.items():
if type(v) in [ray.tune.sample.Float, ray.tune.sample.Integer]:
print(k, (v.lower, v.upper))
phi1 = 1.4962
phi2 = 1.4962
omega = 0.7298
if 'options' in config:
phi1 = config['options']['c1']
phi2 = config['options']['c2']
omega = config['options']['w']
current_best_params = []
if current_best is not None:
if type(current_best) == list:
for c in current_best:
c = clean_start_config(c, config)
if c not in current_best_params:
current_best_params.append(c)
else:
current_best = clean_start_config(current_best, config)
current_best_params.append(current_best)
ray.init(num_cpus=config['num_cpus'],
object_store_memory=memory if memory > 4000000000 else
None) # , logging_level=logging.FATAL, log_to_driver=False)
pso = ng.optimizers.ConfiguredPSO(transform='identity',
popsize=config['n_particles'],
omega=omega,
phip=phi1,
phig=phi2)
algo = NevergradSearch(optimizer=pso,
points_to_evaluate=current_best_params)
algo = ConcurrencyLimiter(algo, max_concurrent=config['num_cpus'])
scheduler = AsyncHyperBandScheduler()
print('\n\nsimple sliding window optimization\n\n')
parameter_columns = []
for side in ['long', 'shrt']:
if config[f'{side}£enabled']:
parameter_columns.append(f'{side}£grid_span')
parameter_columns.append(f'{side}£eprice_pprice_diff')
parameter_columns.append(f'{side}£eprice_exp_base')
parameter_columns.append(f'{side}£secondary_pprice_diff')
parameter_columns.append(f'{side}£min_markup')
backtest_wrap = tune.with_parameters(
simple_sliding_window_wrap,
data=data,
do_print=(config['print_slice_progress']
if 'print_slice_progress' in config else False))
analysis = tune.run(
backtest_wrap,
metric='obj',
mode='max',
name='search',
search_alg=algo,
scheduler=scheduler,
num_samples=config['iters'],
config=config,
verbose=1,
reuse_actors=True,
local_dir=config['optimize_dirpath'],
progress_reporter=LogReporter(metric_columns=[
'min_adg', 'avg_adg', 'min_bkr', 'eqbal_ratio_min',
'hrs_stuck_max_l', 'hrs_stuck_max_s', 'pac_mean_l', 'pac_mean_s',
'n_slc', 'obj'
],
parameter_columns=parameter_columns,
max_report_frequency=30),
raise_on_failed_trial=False)
ray.shutdown()
print('\nCleaning up temporary optimizer data...\n')
try:
shutil.rmtree(os.path.join(config['optimize_dirpath'], 'search'))
except Exception as e:
print('Failed cleaning up.')
print(e)
return analysis
from ray.tune.suggest.nevergrad import NevergradSearch
budget = 400
opt_args = {
'scheduler.target_lrs': ng.var.Log(1e-4, 1e-2),
'scheduler.final_lr': ng.var.Log(1e-7, 1e-4),
'optimizer.weight_decay': ng.var.Log(1e-7, 1e-3),
'scheduler.exp_factor': ng.var.Scalar().bounded(-.5, 2),
}
inst = ng.Instrumentation(**opt_args)
opt = ng.optimizers.registry['CMA'](instrumentation=inst, budget=budget)
search_alg = NevergradSearch(opt,
None,
max_concurrent=4,
reward_attr='valid_accuracy',
mode='max')
exp_args = {
'resume': False,
'search_alg': search_alg,
'scheduler': schedulers.FIFOScheduler(),
'stop': {
'is_finished': True
},
'verbose': True,
'num_samples': budget,
'checkpoint_at_end': False,
'resources_per_trial': {
'cpu': 2,
def _test_xgboost(method='BlendSearch'):
try:
import ray
except ImportError:
return
if method == 'BlendSearch':
from flaml import tune
else:
from ray import tune
search_space = {
# You can mix constants with search space objects.
"max_depth": tune.randint(1, 8) if method in [
"BlendSearch", "BOHB", "Optuna"] else tune.randint(1, 9),
"min_child_weight": tune.choice([1, 2, 3]),
"subsample": tune.uniform(0.5, 1.0),
"eta": tune.loguniform(1e-4, 1e-1)
}
max_iter = 10
for num_samples in [256]:
time_budget_s = 60 #None
for n_cpu in [8]:
start_time = time.time()
ray.init(num_cpus=n_cpu, num_gpus=0)
if method == 'BlendSearch':
analysis = tune.run(
train_breast_cancer,
init_config={
"max_depth": 1,
"min_child_weight": 3,
},
cat_hp_cost={
"min_child_weight": [6, 3, 2],
},
metric="eval-logloss",
mode="min",
max_resource=max_iter,
min_resource=1,
report_intermediate_result=True,
# You can add "gpu": 0.1 to allocate GPUs
resources_per_trial={"cpu": 1},
config=search_space,
local_dir='logs/',
num_samples=num_samples*n_cpu,
time_budget_s=time_budget_s,
use_ray=True)
else:
if 'ASHA' == method:
algo = None
elif 'BOHB' == method:
from ray.tune.schedulers import HyperBandForBOHB
from ray.tune.suggest.bohb import TuneBOHB
algo = TuneBOHB(max_concurrent=n_cpu)
scheduler = HyperBandForBOHB(max_t=max_iter)
elif 'Optuna' == method:
from ray.tune.suggest.optuna import OptunaSearch
algo = OptunaSearch()
elif 'CFO' == method:
from flaml import CFO
algo = CFO(points_to_evaluate=[{
"max_depth": 1,
"min_child_weight": 3,
}], cat_hp_cost={
"min_child_weight": [6, 3, 2],
})
elif 'Dragonfly' == method:
from ray.tune.suggest.dragonfly import DragonflySearch
algo = DragonflySearch()
elif 'SkOpt' == method:
from ray.tune.suggest.skopt import SkOptSearch
algo = SkOptSearch()
elif 'Nevergrad' == method:
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne)
elif 'ZOOpt' == method:
from ray.tune.suggest.zoopt import ZOOptSearch
algo = ZOOptSearch(budget=num_samples*n_cpu)
elif 'Ax' == method:
from ray.tune.suggest.ax import AxSearch
algo = AxSearch()
elif 'HyperOpt' == method:
from ray.tune.suggest.hyperopt import HyperOptSearch
algo = HyperOptSearch()
scheduler = None
if method != 'BOHB':
from ray.tune.schedulers import ASHAScheduler
scheduler = ASHAScheduler(
max_t=max_iter,
grace_period=1)
analysis = tune.run(
train_breast_cancer,
metric="eval-logloss",
mode="min",
# You can add "gpu": 0.1 to allocate GPUs
resources_per_trial={"cpu": 1},
config=search_space, local_dir='logs/',
num_samples=num_samples*n_cpu, time_budget_s=time_budget_s,
scheduler=scheduler, search_alg=algo)
ray.shutdown()
# # Load the best model checkpoint
# best_bst = xgb.Booster()
# best_bst.load_model(os.path.join(analysis.best_checkpoint,
# "model.xgb"))
best_trial = analysis.get_best_trial("eval-logloss","min","all")
accuracy = 1. - best_trial.metric_analysis["eval-error"]["min"]
logloss = best_trial.metric_analysis["eval-logloss"]["min"]
logger.info(f"method={method}")
logger.info(f"n_samples={num_samples*n_cpu}")
logger.info(f"time={time.time()-start_time}")
logger.info(f"Best model eval loss: {logloss:.4f}")
logger.info(f"Best model total accuracy: {accuracy:.4f}")
logger.info(f"Best model parameters: {best_trial.config}")
def _test_distillbert(method='BlendSearch'):
max_num_epoch = 64
num_samples = -1
time_budget_s = 10800
search_space = {
# You can mix constants with search space objects.
"num_train_epochs": flaml.tune.loguniform(1, max_num_epoch),
"learning_rate": flaml.tune.loguniform(1e-6, 1e-4),
"adam_beta1": flaml.tune.uniform(0.8, 0.99),
"adam_beta2": flaml.tune.loguniform(98e-2, 9999e-4),
"adam_epsilon": flaml.tune.loguniform(1e-9, 1e-7),
}
start_time = time.time()
ray.init(num_cpus=4, num_gpus=4)
if 'ASHA' == method:
algo = None
elif 'BOHB' == method:
from ray.tune.schedulers import HyperBandForBOHB
from ray.tune.suggest.bohb import tuneBOHB
algo = tuneBOHB(max_concurrent=4)
scheduler = HyperBandForBOHB(max_t=max_num_epoch)
elif 'Optuna' == method:
from ray.tune.suggest.optuna import OptunaSearch
algo = OptunaSearch()
elif 'CFO' == method:
from flaml import CFO
algo = CFO(points_to_evaluate=[{
"num_train_epochs": 1,
}])
elif 'BlendSearch' == method:
from flaml import BlendSearch
algo = BlendSearch(points_to_evaluate=[{
"num_train_epochs": 1,
}])
elif 'Dragonfly' == method:
from ray.tune.suggest.dragonfly import DragonflySearch
algo = DragonflySearch()
elif 'SkOpt' == method:
from ray.tune.suggest.skopt import SkOptSearch
algo = SkOptSearch()
elif 'Nevergrad' == method:
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne)
elif 'ZOOpt' == method:
from ray.tune.suggest.zoopt import ZOOptSearch
algo = ZOOptSearch(budget=num_samples)
elif 'Ax' == method:
from ray.tune.suggest.ax import AxSearch
algo = AxSearch()
elif 'HyperOpt' == method:
from ray.tune.suggest.hyperopt import HyperOptSearch
algo = HyperOptSearch()
scheduler = None
if method != 'BOHB':
from ray.tune.schedulers import ASHAScheduler
scheduler = ASHAScheduler(max_t=max_num_epoch, grace_period=1)
scheduler = None
analysis = ray.tune.run(
train_distilbert,
metric=HP_METRIC,
mode=MODE,
# You can add "gpu": 1 to allocate GPUs
resources_per_trial={"gpu": 1},
config=search_space,
local_dir='test/logs/',
num_samples=num_samples,
time_budget_s=time_budget_s,
keep_checkpoints_num=1,
checkpoint_score_attr=HP_METRIC,
scheduler=scheduler,
search_alg=algo)
ray.shutdown()
best_trial = analysis.get_best_trial(HP_METRIC, MODE, "all")
metric = best_trial.metric_analysis[HP_METRIC][MODE]
logger.info(f"method={method}")
logger.info(f"n_trials={len(analysis.trials)}")
logger.info(f"time={time.time()-start_time}")
logger.info(f"Best model eval {HP_METRIC}: {metric:.4f}")
logger.info(f"Best model parameters: {best_trial.config}")
def _test_roberta(method='BlendSearch'):
max_num_epoch = 100
num_samples = -1
time_budget_s = 3600
search_space = {
# You can mix constants with search space objects.
"num_train_epochs": flaml.tune.loguniform(1, max_num_epoch),
"learning_rate": flaml.tune.loguniform(1e-5, 3e-5),
"weight_decay": flaml.tune.uniform(0, 0.3),
"per_device_train_batch_size": flaml.tune.choice([16, 32, 64, 128]),
"seed": flaml.tune.choice([12, 22, 33, 42]),
}
start_time = time.time()
ray.init(num_cpus=4, num_gpus=4)
if 'ASHA' == method:
algo = None
elif 'BOHB' == method:
from ray.tune.schedulers import HyperBandForBOHB
from ray.tune.suggest.bohb import tuneBOHB
algo = tuneBOHB(max_concurrent=4)
scheduler = HyperBandForBOHB(max_t=max_num_epoch)
elif 'Optuna' == method:
from ray.tune.suggest.optuna import OptunaSearch
algo = OptunaSearch()
elif 'CFO' == method:
from flaml import CFO
algo = CFO(points_to_evaluate=[{
"num_train_epochs": 1,
"per_device_train_batch_size": 128,
}])
elif 'BlendSearch' == method:
from flaml import BlendSearch
algo = BlendSearch(
points_to_evaluate=[{
"num_train_epochs": 1,
"per_device_train_batch_size": 128,
}])
elif 'Dragonfly' == method:
from ray.tune.suggest.dragonfly import DragonflySearch
algo = DragonflySearch()
elif 'SkOpt' == method:
from ray.tune.suggest.skopt import SkOptSearch
algo = SkOptSearch()
elif 'Nevergrad' == method:
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne)
elif 'ZOOpt' == method:
from ray.tune.suggest.zoopt import ZOOptSearch
algo = ZOOptSearch(budget=num_samples)
elif 'Ax' == method:
from ray.tune.suggest.ax import AxSearch
algo = AxSearch(max_concurrent=3)
elif 'HyperOpt' == method:
from ray.tune.suggest.hyperopt import HyperOptSearch
algo = HyperOptSearch()
scheduler = None
if method != 'BOHB':
from ray.tune.schedulers import ASHAScheduler
scheduler = ASHAScheduler(max_t=max_num_epoch, grace_period=1)
scheduler = None
analysis = ray.tune.run(train_roberta,
metric=HP_METRIC,
mode=MODE,
resources_per_trial={
"gpu": 4,
"cpu": 4
},
config=search_space,
local_dir='logs/',
num_samples=num_samples,
time_budget_s=time_budget_s,
keep_checkpoints_num=1,
checkpoint_score_attr=HP_METRIC,
scheduler=scheduler,
search_alg=algo)
ray.shutdown()
best_trial = analysis.get_best_trial(HP_METRIC, MODE, "all")
metric = best_trial.metric_analysis[HP_METRIC][MODE]
logger.info(f"method={method}")
logger.info(f"n_trials={len(analysis.trials)}")
logger.info(f"time={time.time()-start_time}")
logger.info(f"Best model eval {HP_METRIC}: {metric:.4f}")
logger.info(f"Best model parameters: {best_trial.config}")
args, _ = parser.parse_known_args()
if args.server_address:
import ray
ray.init(f"ray://{args.server_address}")
# Optional: Pass the parameter space yourself
# space = ng.p.Dict(
# width=ng.p.Scalar(lower=0, upper=20),
# height=ng.p.Scalar(lower=-100, upper=100),
# activation=ng.p.Choice(choices=["relu", "tanh"])
# )
algo = NevergradSearch(
optimizer=ng.optimizers.OnePlusOne,
# space=space, # If you want to set the space manually
)
algo = ConcurrencyLimiter(algo, max_concurrent=4)
scheduler = AsyncHyperBandScheduler()
analysis = tune.run(
easy_objective,
metric="mean_loss",
mode="min",
name="nevergrad",
search_alg=algo,
scheduler=scheduler,
num_samples=10 if args.smoke_test else 50,
config={
"steps": 100,
