Exemplo n.º 1
0
    def testWithParameters(self):
        class Data:
            def __init__(self):
                self.data = [0] * 500_000

        data = Data()
        data.data[100] = 1

        class TestTrainable(Trainable):
            def setup(self, config, data):
                self.data = data.data
                self.data[101] = 2  # Changes are local

            def step(self):
                return dict(
                    metric=len(self.data), hundred=self.data[100], done=True)

        trial_1, trial_2 = tune.run(
            tune.with_parameters(TestTrainable, data=data),
            num_samples=2).trials

        self.assertEqual(data.data[101], 0)
        self.assertEqual(trial_1.last_result["metric"], 500_000)
        self.assertEqual(trial_1.last_result["hundred"], 1)
        self.assertEqual(trial_2.last_result["metric"], 500_000)
        self.assertEqual(trial_2.last_result["hundred"], 1)
        self.assertTrue(str(trial_1).startswith("TestTrainable"))
Exemplo n.º 2
0
def main(cpus_per_actor, num_actors, num_samples):
    # Set XGBoost config.
    config = {
        "tree_method": "approx",
        "objective": "binary:logistic",
        "eval_metric": ["logloss", "error"],
        "eta": tune.loguniform(1e-4, 1e-1),
        "subsample": tune.uniform(0.5, 1.0),
        "max_depth": tune.randint(1, 9)
    }

    ray_params = RayParams(max_actor_restarts=1,
                           gpus_per_actor=0,
                           cpus_per_actor=cpus_per_actor,
                           num_actors=num_actors)

    analysis = tune.run(
        tune.with_parameters(train_breast_cancer, ray_params=ray_params),
        # Use the `get_tune_resources` helper function to set the resources.
        resources_per_trial=ray_params.get_tune_resources(),
        config=config,
        num_samples=num_samples,
        metric="eval-error",
        mode="min")

    # Load the best model checkpoint.
    best_bst = xgboost_ray.tune.load_model(
        os.path.join(analysis.best_logdir, "tuned.xgb"))

    best_bst.save_model("best_model.xgb")

    accuracy = 1. - analysis.best_result["eval-error"]
    print(f"Best model parameters: {analysis.best_config}")
    print(f"Best model total accuracy: {accuracy:.4f}")
Exemplo n.º 3
0
    def testWithParametersTwoRuns2(self):
        # Makes sure two runs in the same script
        # pass (https://github.com/ray-project/ray/issues/16609)
        def train_fn(config, extra=4):
            tune.report(metric=extra)

        def train_fn_2(config, extra=5):
            tune.report(metric=extra)

        trainable1 = tune.with_parameters(train_fn, extra=8)
        trainable2 = tune.with_parameters(train_fn_2, extra=9)

        out1 = tune.run(trainable1, metric="metric", mode="max")
        out2 = tune.run(trainable2, metric="metric", mode="max")
        self.assertEquals(out1.best_result["metric"], 8)
        self.assertEquals(out2.best_result["metric"], 9)
Exemplo n.º 4
0
def run_tuning_procedure(config,
                         expname,
                         ntrials,
                         ncpus,
                         ngpus,
                         NetClass,
                         dataset="hchs"):

    trainable = tune.with_parameters(hyper_tuner,
                                     NetClass=NetClass,
                                     dataset=dataset)

    analysis = tune.run(trainable,
                        resources_per_trial={
                            "cpu": ncpus,
                            "gpu": ngpus
                        },
                        metric="loss",
                        mode="min",
                        config=config,
                        num_samples=ntrials,
                        name=expname)

    print("Best Parameters:", analysis.best_config)

    analysis.best_result_df.to_csv("best_parameters_exp%s_trials%d.csv" %
                                   (expname, ntrials))
    analysis.results_df.to_csv("all_results_exp%s_trials%d.csv" %
                               (expname, ntrials))
    print("Best 5 results")
    print(analysis.results_df.sort_values(by="mcc", ascending=False).head(5))
Exemplo n.º 5
0
def main_tune(base_args):
    # ray.init(log_to_driver=False)
    tune_config = {
        "learning_rate": tune.loguniform(5e-6, 1e-3),
        "weight_decay": tune.choice([0.0, 1e-3, 1e-2, 0.1]),
        "batch_size": tune.choice([16, 32, 64, 128]),
        "latent_dim": tune.choice([2, 3, 8, 16, 32, 128, 256, 512])
    }

    scheduler = ASHAScheduler(max_t=base_args.max_tune_epoches,
                              grace_period=3,
                              reduction_factor=2)

    reporter = CLIReporter(parameter_columns=[
        "learning_rate", "weight_decay", "batch_size", "latent_dim"
    ],
                           metric_columns=[
                               "val_lossR", "loss", "Reconstruction_Loss",
                               "training_iteration"
                           ])

    analysis = tune.run(tune.with_parameters(tune_train, base_arg=base_args),
                        resources_per_trial={
                            "cpu": 12,
                            "gpu": 1.0,
                        },
                        metric="val_lossR",
                        mode="min",
                        config=tune_config,
                        num_samples=10,
                        scheduler=scheduler,
                        progress_reporter=reporter,
                        name="tune_vae_chol")

    print("Best hyperparameters found were: ", analysis.best_config)
Exemplo n.º 6
0
def main(cpus_per_actor, num_actors, num_samples):
    # Set XGBoost config.
    config = {
        "tree_method": "approx",
        "objective": "binary:logistic",
        "eval_metric": ["logloss", "error"],
        "eta": tune.loguniform(1e-4, 1e-1),
        "subsample": tune.uniform(0.5, 1.0),
        "max_depth": tune.randint(1, 9)
    }

    analysis = tune.run(
        tune.with_parameters(train_breast_cancer,
                             cpus_per_actor=cpus_per_actor,
                             num_actors=num_actors),
        # extra_cpu is used if the trainable creates additional remote actors.
        # https://docs.ray.io/en/master/tune/api_docs/trainable.html#advanced-resource-allocation
        resources_per_trial={
            "cpu": 1,
            "extra_cpu": cpus_per_actor * num_actors
        },
        config=config,
        num_samples=num_samples,
        metric="eval-error",
        mode="min")

    # Load the best model checkpoint
    best_bst = xgb.Booster()
    best_bst.load_model(os.path.join(analysis.best_logdir, "simple.xgb"))
    accuracy = 1. - analysis.best_result["eval-error"]
    print(f"Best model parameters: {analysis.best_config}")
    print(f"Best model total accuracy: {accuracy:.4f}")
Exemplo n.º 7
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--gpus")
    parser.add_argument("--gpus-per-trial", type=float)
    parser.add_argument("--num-epochs", type=int)
    parser.add_argument("--num-samples", type=int)
    parser.add_argument("--w2v", type=str)
    args = parser.parse_args()

    w2v_sd = torch.load(args.w2v)
    gpus_per_trial = args.gpus_per_trial
    trainable = tune.with_parameters(
        train_model,
        gpus=args.gpus,
        w2v=w2v_sd,
        num_epochs=args.num_epochs,
    )

    algo = AxSearch(max_concurrent=4)
    scheduler = AsyncHyperBandScheduler()

    analysis = tune.run(trainable,
                        resources_per_trial={
                            "cpu": 4,
                            "gpu": gpus_per_trial
                        },
                        metric="acc",
                        mode="max",
                        search_alg=algo,
                        scheduler=scheduler,
                        config=config,
                        num_samples=args.num_samples,
                        name="tune_w2v_lr")

    print(analysis.best_config)
Exemplo n.º 8
0
def main(num_samples=10, max_num_epochs=10, gpus_per_trial=2):
    config = {
        "l1": tune.sample_from(lambda _: 2 ** np.random.randint(2, 9)),
        "l2": tune.sample_from(lambda _: 2 ** np.random.randint(2, 9)),
        "lr": tune.loguniform(1e-4, 1e-1),
        "batch_size": tune.choice([2, 4, 8, 16])
    }
    scheduler = ASHAScheduler(
        max_t=max_num_epochs,
        grace_period=1,
        reduction_factor=2)
    result = tune.run(
        tune.with_parameters(train_cifar),
        resources_per_trial={"cpu": 2, "gpu": gpus_per_trial},
        config=config,
        metric="loss",
        mode="min",
        num_samples=num_samples,
        scheduler=scheduler
    )

    best_trial = result.get_best_trial("loss", "min", "last")
    print("Best trial config: {}".format(best_trial.config))
    print("Best trial final validation loss: {}".format(
        best_trial.last_result["loss"]))
    print("Best trial final validation accuracy: {}".format(
        best_trial.last_result["accuracy"]))

    if ray.util.client.ray.is_connected():
        # If using Ray Client, we want to make sure checkpoint access
        # happens on the server. So we wrap `test_best_model` in a Ray task.
        ray.get(ray.remote(test_best_model).remote(best_trial))
    else:
        test_best_model(best_trial)
Exemplo n.º 9
0
def tune_mnist(data_dir,
               num_samples=10,
               num_epochs=10,
               num_workers=1,
               use_gpu=False):
    config = {
        "layer_1": tune.choice([32, 64, 128]),
        "layer_2": tune.choice([64, 128, 256]),
        "lr": tune.loguniform(1e-4, 1e-1),
        "batch_size": tune.choice([32, 64, 128]),
    }

    # Add Tune callback.
    metrics = {"loss": "ptl/val_loss", "acc": "ptl/val_accuracy"}
    callbacks = [TuneReportCallback(metrics, on="validation_end")]
    trainable = tune.with_parameters(train_mnist,
                                     data_dir=data_dir,
                                     num_epochs=num_epochs,
                                     num_workers=num_workers,
                                     use_gpu=use_gpu,
                                     callbacks=callbacks)
    analysis = tune.run(trainable,
                        metric="loss",
                        mode="min",
                        config=config,
                        num_samples=num_samples,
                        resources_per_trial={
                            "cpu": 1,
                            "gpu": int(use_gpu),
                            "extra_cpu": num_workers,
                            "extra_gpu": num_workers * int(use_gpu)
                        },
                        name="tune_mnist")

    print("Best hyperparameters found were: ", analysis.best_config)
Exemplo n.º 10
0
def tune_mnist_mxnet(num_samples=10, num_epochs=10):
    logger.info("Downloading MNIST data...")
    mnist_data = mx.test_utils.get_mnist()
    logger.info("Got MNIST data, starting Ray Tune.")

    config = {
        "layer_1_size": tune.choice([32, 64, 128]),
        "layer_2_size": tune.choice([64, 128, 256]),
        "lr": tune.loguniform(1e-3, 1e-1),
        "batch_size": tune.choice([32, 64, 128]),
    }

    scheduler = ASHAScheduler(max_t=num_epochs, grace_period=1, reduction_factor=2)

    analysis = tune.run(
        tune.with_parameters(
            train_mnist_mxnet, mnist=mnist_data, num_epochs=num_epochs
        ),
        resources_per_trial={
            "cpu": 1,
        },
        metric="mean_accuracy",
        mode="max",
        config=config,
        num_samples=num_samples,
        scheduler=scheduler,
        name="tune_mnist_mxnet",
    )
    return analysis
Exemplo n.º 11
0
def tune_mnist(num_samples=10, num_epochs=10, gpus_per_trial=0):
    data_dir = os.path.join(tempfile.gettempdir(), "mnist_data_")
    # Download data
    MNISTDataModule(data_dir=data_dir).prepare_data()

    config = {
        "layer_1": tune.choice([32, 64, 128]),
        "layer_2": tune.choice([64, 128, 256]),
        "lr": tune.loguniform(1e-4, 1e-1),
        "batch_size": tune.choice([32, 64, 128]),
    }

    trainable = tune.with_parameters(train_mnist_tune,
                                     data_dir=data_dir,
                                     num_epochs=num_epochs,
                                     num_gpus=gpus_per_trial)
    analysis = tune.run(trainable,
                        resources_per_trial={
                            "cpu": 1,
                            "gpu": gpus_per_trial
                        },
                        metric="loss",
                        mode="min",
                        config=config,
                        num_samples=num_samples,
                        name="tune_mnist")

    print("Best hyperparameters found were: ", analysis.best_config)
Exemplo n.º 12
0
def main(num_samples=10, max_num_epochs=15):
    # data_dir = os.path.abspath("./data")
    configs = load_configs("configs/training/smg_configs_template.yaml")
    # configs["processor"]["torch_model_dict"]["hidden_sizes"] =
    a = tune.choice([[90, 45, 25, 10], [45, 25], [45, 25], [90, 45, 25, 10]])
    dataloaders, amplification, data_info_dict = load_data(
        configs)  # Download data for all trials before starting the run

    scheduler = ASHAScheduler(max_t=max_num_epochs,
                              grace_period=1,
                              reduction_factor=2)
    result = tune.run(
        tune.with_parameters(generate_surrogate_model, configs=configs),
        resources_per_trial={
            "cpu": 8,
            "gpu": 1
        },
        config=configs,
        metric="loss",
        mode="min",
        num_samples=num_samples,
        scheduler=scheduler,
    )

    best_trial = result.get_best_trial("loss", "min", "last")
    print("Best trial config: {}".format(best_trial.config))
    print("Best trial final validation loss: {}".format(
        best_trial.last_result["loss"]))
    print("Best trial final validation accuracy: {}".format(
        best_trial.last_result["accuracy"]))
Exemplo n.º 13
0
def grid_search(hparams):
    scheduler = ASHAScheduler(max_t=hparams['n_epochs'],
                              grace_period=1,
                              reduction_factor=2)

    reporter = CLIReporter(
        parameter_columns=hparams['param_cols'],
        metric_columns=['valid_acc', 'valid_f1', 'valid_loss'])

    rdm = RetinalDataModule()

    analysis = tune.run(tune.with_parameters(train_tune, rdm=rdm),
                        resources_per_trial={
                            "cpu": 1,
                            "gpu": 1
                        },
                        metric="valid_loss",
                        mode="min",
                        config=hparams,
                        local_dir=Path(hparams['output_dir'], 'ray_tune'),
                        num_samples=5,
                        scheduler=scheduler,
                        progress_reporter=reporter,
                        name=f"tune_{hparams['model']}_DRIVE")

    print("Best hyperparameters found were: ", analysis.best_config)
Exemplo n.º 14
0
def tune_xgboost(train_df, test_df, target_column):
    # Set XGBoost config.
    config = {
        "tree_method": "approx",
        "objective": "binary:logistic",
        "eval_metric": ["logloss", "error"],
        "eta": tune.loguniform(1e-4, 1e-1),
        "subsample": tune.uniform(0.5, 1.0),
        "max_depth": tune.randint(1, 9)
    }

    ray_params = RayParams(max_actor_restarts=1,
                           gpus_per_actor=0,
                           cpus_per_actor=4,
                           num_actors=4)

    analysis = tune.run(
        tune.with_parameters(train_xgboost,
                             train_df=train_df,
                             test_df=test_df,
                             target_column=target_column,
                             ray_params=ray_params),
        # Use the `get_tune_resources` helper function to set the resources.
        resources_per_trial=ray_params.get_tune_resources(),
        config=config,
        num_samples=1,
        metric="eval-error",
        mode="min",
        verbose=1)

    accuracy = 1. - analysis.best_result["eval-error"]
    print(f"Best model parameters: {analysis.best_config}")
    print(f"Best model total accuracy: {accuracy:.4f}")

    return analysis.best_config
Exemplo n.º 15
0
def hypertune(num_samples, num_epochs, sym01, sym02, period):
    config = {
        "seq_len": tune.choice([5, 10]),
        "hidden_size": tune.choice([10, 50, 100]),
        "batch_size": tune.choice([30, 60]),
        "dropout": tune.choice([0.1, 0.2]),
        "lr": tune.loguniform(1e-4, 1e-1),
        "num_layers": tune.choice([2, 3, 4])
    }
    trainable = tune.with_parameters(
        myTrain,
        num_epochs=num_epochs,
        sym01=sym01,
        sym02=sym02,
        period=period,
    )
    analysis = tune.run(trainable,
                        resources_per_trial={
                            "cpu": 1,
                        },
                        metric="loss",
                        mode="min",
                        config=config,
                        num_samples=num_samples,
                        name="tune_lstm")
    print("tuning finished")
    return analysis.best_config
Exemplo n.º 16
0
def tune_mnist(num_samples=10, num_epochs=10, gpus_per_trial=0):
    config = {
        "layer_1": tune.choice([32, 64, 128]),
        "layer_2": tune.choice([64, 128, 256]),
        "lr": tune.loguniform(1e-4, 1e-1),
        "batch_size": tune.choice([32, 64, 128]),
    }

    trainable = tune.with_parameters(train_mnist_tune,
                                     num_epochs=num_epochs,
                                     num_gpus=gpus_per_trial)
    analysis = tune.run(
        trainable,
        resources_per_trial={
            "cpu": 1,
            "gpu": gpus_per_trial
        },
        metric="loss",
        mode="min",
        config=config,
        num_samples=num_samples,
        name="tune_mnist",
    )

    print("Best hyperparameters found were: ", analysis.best_config)
Exemplo n.º 17
0
def tune_mnist_asha(num_samples=10, num_epochs=10, gpus_per_trial=0, data_dir="~/data"):
    config = {
        "layer_1_size": tune.choice([32, 64, 128]),
        "layer_2_size": tune.choice([64, 128, 256]),
        "lr": tune.loguniform(1e-4, 1e-1),
        "batch_size": tune.choice([32, 64, 128]),
    }

    scheduler = ASHAScheduler(
        max_t=num_epochs,
        grace_period=1,
        reduction_factor=2)

    reporter = CLIReporter(
        parameter_columns=["layer_1_size", "layer_2_size", "lr", "batch_size"],
        metric_columns=["loss", "mean_accuracy", "training_iteration"])

    train_fn_with_parameters = tune.with_parameters(train_mnist_tune,
                                                    num_epochs=num_epochs,
                                                    num_gpus=gpus_per_trial,
                                                    data_dir=data_dir)
    resources_per_trial = {"cpu": 1, "gpu": gpus_per_trial}

    analysis = tune.run(train_fn_with_parameters,
        resources_per_trial=resources_per_trial,
        metric="loss",
        mode="min",
        config=config,
        num_samples=num_samples,
        scheduler=scheduler,
        progress_reporter=reporter,
        name="tune_mnist_asha")

    print("Best hyperparameters found were: ", analysis.best_config)
Exemplo n.º 18
0
def backtest_tune(ticks: np.ndarray, backtest_config: dict, current_best: Union[dict, list] = None):
    config = create_config(backtest_config)
    n_days = round_((ticks[-1][2] - ticks[0][2]) / (1000 * 60 * 60 * 24), 0.1)
    session_dirpath = make_get_filepath(os.path.join('reports', backtest_config['exchange'], backtest_config['symbol'],
                                                     f"{n_days}_days_{ts_to_date(time())[:19].replace(':', '')}", ''))
    iters = 10
    if 'iters' in backtest_config:
        iters = backtest_config['iters']
    else:
        print('Parameter iters should be defined in the configuration. Defaulting to 10.')
    num_cpus = 2
    if 'num_cpus' in backtest_config:
        num_cpus = backtest_config['num_cpus']
    else:
        print('Parameter num_cpus should be defined in the configuration. Defaulting to 2.')
    n_particles = 10
    if 'n_particles' in backtest_config:
        n_particles = backtest_config['n_particles']
    phi1 = 1.4962
    phi2 = 1.4962
    omega = 0.7298
    if 'options' in backtest_config:
        phi1 = backtest_config['options']['c1']
        phi2 = backtest_config['options']['c2']
        omega = backtest_config['options']['w']
    current_best_params = []
    if current_best:
        if type(current_best) == list:
            for c in current_best:
                c = clean_start_config(c, config, backtest_config['ranges'])
                current_best_params.append(c)
        else:
            current_best = clean_start_config(current_best, config, backtest_config['ranges'])
            current_best_params.append(current_best)

    ray.init(num_cpus=num_cpus, logging_level=logging.FATAL, log_to_driver=False)
    pso = ng.optimizers.ConfiguredPSO(transform='identity', popsize=n_particles, omega=omega, phip=phi1, phig=phi2)
    algo = NevergradSearch(optimizer=pso, points_to_evaluate=current_best_params)
    algo = ConcurrencyLimiter(algo, max_concurrent=num_cpus)
    scheduler = AsyncHyperBandScheduler()

    analysis = tune.run(tune.with_parameters(backtest, ticks=ticks), metric='objective', mode='max', name='search',
                        search_alg=algo, scheduler=scheduler, num_samples=iters, config=config, verbose=1,
                        reuse_actors=True, local_dir=session_dirpath,
                        progress_reporter=LogReporter(metric_columns=['daily_gain', 'closest_liquidation', 'objective'],
                                                      parameter_columns=[k for k in backtest_config['ranges']]))

    ray.shutdown()
    df = analysis.results_df
    df.reset_index(inplace=True)
    df.drop(columns=['trial_id', 'time_this_iter_s', 'done', 'timesteps_total', 'episodes_total', 'training_iteration',
                     'experiment_id', 'date', 'timestamp', 'time_total_s', 'pid', 'hostname', 'node_ip',
                     'time_since_restore', 'timesteps_since_restore', 'iterations_since_restore', 'experiment_tag'],
            inplace=True)
    df.to_csv(os.path.join(backtest_config['session_dirpath'], 'results.csv'), index=False)
    print('Best candidate found:')
    pprint.pprint(analysis.best_config)
    plot_wrap(backtest_config, ticks, clean_result_config(analysis.best_config))
    return analysis
Exemplo n.º 19
0
def main():
    name = "large xgboost sweep"

    ray.init(address="auto")

    num_samples = 31  # So that we fit on 1024 CPUs with 1 head bundle
    num_actors_per_sample = 32

    max_runtime = 3500

    config = {
        "tree_method": "approx",
        "objective": "binary:logistic",
        "eval_metric": ["logloss", "error"],
        "eta": tune.loguniform(1e-4, 1e-1),
        "subsample": tune.uniform(0.5, 1.0),
        "max_depth": 4,
    }

    ray_params = RayParams(
        max_actor_restarts=1,
        gpus_per_actor=0,
        cpus_per_actor=1,
        num_actors=num_actors_per_sample,
    )

    start_time = time.monotonic()
    analysis = tune.run(
        tune.with_parameters(xgboost_train,
                             ray_params=ray_params,
                             num_boost_round=100),
        config=config,
        num_samples=num_samples,
        resources_per_trial=ray_params.get_tune_resources(),
    )
    time_taken = time.monotonic() - start_time

    result = {
        "time_taken": time_taken,
        "trial_states":
        dict(Counter([trial.status for trial in analysis.trials])),
        "last_update": time.time(),
    }
    test_output_json = os.environ.get("TEST_OUTPUT_JSON",
                                      "/tmp/tune_test.json")
    with open(test_output_json, "wt") as f:
        json.dump(result, f)

    if time_taken > max_runtime:
        print(f"The {name} test took {time_taken:.2f} seconds, but should not "
              f"have exceeded {max_runtime:.2f} seconds. Test failed. \n\n"
              f"--- FAILED: {name.upper()} ::: "
              f"{time_taken:.2f} > {max_runtime:.2f} ---")
    else:
        print(f"The {name} test took {time_taken:.2f} seconds, which "
              f"is below the budget of {max_runtime:.2f} seconds. "
              f"Test successful. \n\n"
              f"--- PASSED: {name.upper()} ::: "
              f"{time_taken:.2f} <= {max_runtime:.2f} ---")
Exemplo n.º 20
0
def optimize_hyperparameters(
    train_model,
    create_model,
    data_train,
    data_test,
    search_space,
    model_kwargs_str,
    callbacks,
    hyperparams_file_name,
    random_seed,
    model_path,
    epochs,
    n_steps,
    num_samples_optim,
):
    tmp_dir = tempfile.TemporaryDirectory(dir=os.getcwd())

    ray.shutdown()
    ray.init(log_to_driver=False, local_mode=True)

    search_alg = HyperOptSearch(random_state_seed=random_seed)
    search_alg = ConcurrencyLimiter(search_alg, max_concurrent=1)
    scheduler = AsyncHyperBandScheduler(time_attr="training_iteration",
                                        grace_period=10)

    analysis = tune.run(
        tune.with_parameters(
            train_model,
            data_train=data_train,
            data_test=data_test,
            create_model=create_model,
            model_kwargs_str=model_kwargs_str,
            callbacks=callbacks,
            epochs=epochs,
            n_steps=n_steps,
        ),
        verbose=1,
        config=search_space,
        search_alg=search_alg,
        scheduler=scheduler,
        resources_per_trial={
            "cpu": os.cpu_count(),
            "gpu": 0
        },
        metric="val_loss",
        mode="min",
        name="ray_tune_keras_hyperopt_gru",
        local_dir=tmp_dir.name,
        num_samples=num_samples_optim,
    )

    shutil.rmtree(tmp_dir)

    best_params = analysis.get_best_config(metric="val_loss", mode="min")
    with open(os.path.join(model_path, hyperparams_file_name), "w") as f:
        json.dump(best_params, f)
Exemplo n.º 21
0
def backtest_tune(ohlc: np.ndarray, backtest_config: dict):
    config = create_config(backtest_config)
    if not os.path.isdir(os.path.join('reports', backtest_config['symbol'])):
        os.makedirs(os.path.join('reports', backtest_config['symbol']),
                    exist_ok=True)
    report_path = os.path.join('reports', backtest_config['symbol'])
    iters = 10
    if 'iters' in backtest_config:
        iters = backtest_config['iters']
    else:
        print(
            'Parameter iters should be defined in the configuration. Defaulting to 10.'
        )
    num_cpus = 2
    if 'num_cpus' in backtest_config:
        num_cpus = backtest_config['num_cpus']
    else:
        print(
            'Parameter num_cpus should be defined in the configuration. Defaulting to 2.'
        )

    initial_points = max(1, min(int(iters / 10), 20))

    ray.init(num_cpus=num_cpus
             )  # , logging_level=logging.FATAL, log_to_driver=False)

    algo = HyperOptSearch(n_initial_points=initial_points)
    algo = ConcurrencyLimiter(algo, max_concurrent=num_cpus)
    scheduler = AsyncHyperBandScheduler()

    analysis = tune.run(tune.with_parameters(backtest, ohlc=ohlc),
                        metric='objective',
                        mode='max',
                        name='search',
                        search_alg=algo,
                        scheduler=scheduler,
                        num_samples=iters,
                        config=config,
                        verbose=1,
                        reuse_actors=True,
                        local_dir=report_path)

    ray.shutdown()
    session_path = os.path.join(
        os.path.join('sessions', backtest_config['symbol']),
        backtest_config['session_name'])
    if not os.path.isdir(session_path):
        os.makedirs(session_path, exist_ok=True)

    print('Best candidate found is: ', analysis.best_config)
    json.dump(analysis.best_config,
              open(os.path.join(session_path, 'best_config.json'), 'w'),
              indent=4)
    result = backtest(analysis.best_config, ohlc, True)
    result.to_csv(os.path.join(session_path, 'best_trades.csv'), index=False)
    return analysis
Exemplo n.º 22
0
def tune4_withLabel(
    model,
    train_set: Dataset,
    val_set: Dataset,
    dims: list,
    config: dict,
    EPOCHS: int = 300,
    extra_feature_len: int = 0,
    extra_feature_len2: int = 0,
    n_gpu=1,
    n_samples=20,
    model_name="model",
):

    dim1, dim2, dim3, dim4 = dims[0], dims[1], dims[2], dims[3]

    scheduler = ASHAScheduler(max_t=EPOCHS, grace_period=1, reduction_factor=2)
    reporter = CLIReporter(
        parameter_columns=["k", "lr", "batch_size", "hidden_dim"],
        metric_columns=["loss", "training_iteration"],
        max_error_rows=5,
        max_progress_rows=5,
        max_report_frequency=10)

    analysis = tune.run(tune.with_parameters(
        train4_withLabel,
        model=model,
        dim1=dim1,
        dim2=dim2,
        dim3=dim3,
        dim4=dim4,
        extra_feature_len=extra_feature_len,
        extra_feature_len2=extra_feature_len2,
        train_set=train_set,
        val_set=val_set,
        num_epochs=EPOCHS,
        num_gpus=n_gpu,
        model_name=model_name),
                        resources_per_trial={
                            "cpu": 1,
                            "gpu": n_gpu
                        },
                        metric="loss",
                        mode="min",
                        config=config,
                        num_samples=n_samples,
                        scheduler=scheduler,
                        progress_reporter=reporter,
                        name=model_name,
                        verbose=False)

    print("-" * 70)
    print("Done")
    print("Best hyperparameters found were: ", analysis.best_config)
    print("Best achieved loss was: ", analysis.best_result)
    print("-" * 70)
Exemplo n.º 23
0
    def testWithParametersTwoRuns1(self):
        # Makes sure two runs in the same script but different ray sessions
        # pass (https://github.com/ray-project/ray/issues/16609)
        def train_fn(config, extra=4):
            tune.report(metric=extra)

        trainable = tune.with_parameters(train_fn, extra=8)
        out = tune.run(trainable, metric="metric", mode="max")
        self.assertEquals(out.best_result["metric"], 8)

        self.tearDown()
        self.setUp()

        def train_fn_2(config, extra=5):
            tune.report(metric=extra)

        trainable = tune.with_parameters(train_fn_2, extra=9)
        out = tune.run(trainable, metric="metric", mode="max")
        self.assertEquals(out.best_result["metric"], 9)
Exemplo n.º 24
0
def start_training(name):
    Epochs = 1000
    Samples = 50
    ModelName = name

    pose_autoencoder = MLP_withLabel.load_checkpoint(
        "/home/nuoc/Documents/MEX/models/MLP4_withLabel_best/M3/0.00324857.512.pbz2"
    )
    # pose_autoencoder = MLP_withLabel.load_checkpoint("/home/nuoc/Documents/MEX/models/MLP_withLabel/0.0013522337.512.pbz2")

    pose_encoder_out_dim = pose_autoencoder.dimensions[-1]

    scheduler = ASHAScheduler(max_t=Epochs,
                              grace_period=15,
                              reduction_factor=2)
    reporter = CLIReporter(
        parameter_columns=["k", "lr", "batch_size", "loss_fn"],
        metric_columns=["loss", "training_iteration"],
        max_error_rows=5,
        max_progress_rows=5,
        max_report_frequency=1)

    analysis = tune.run(tune.with_parameters(
        tuning,
        MODEL=MotionGenerationModel,
        pose_autoencoder=pose_autoencoder,
        cost_dim=cost_dim,
        phase_dim=phase_dim,
        input_slices=[phase_dim, pose_dim, cost_dim],
        output_slices=[phase_dim, phase_dim, pose_encoder_out_dim],
        train_set=train_set,
        val_set=val_set,
        num_epochs=Epochs,
        model_name=ModelName),
                        resources_per_trial={
                            "cpu": 2,
                            "gpu": 1
                        },
                        metric="loss",
                        mode="min",
                        config=config,
                        num_samples=Samples,
                        scheduler=scheduler,
                        progress_reporter=reporter,
                        name=ModelName,
                        verbose=False)

    print("-" * 70)
    print("Done")
    print("Best hyperparameters found were: ", analysis.best_config)
    print("Best achieved loss was: ", analysis.best_result)
    print("-" * 70)

    ray.shutdown()
Exemplo n.º 25
0
def backtest_tune(ticks: np.ndarray, backtest_config: dict, current_best: Union[dict, list] = None):
    config = create_config(backtest_config)
    n_days = round_((ticks[-1][2] - ticks[0][2]) / (1000 * 60 * 60 * 24), 0.1)
    session_dirpath = make_get_filepath(os.path.join('reports', backtest_config['exchange'], backtest_config['symbol'],
                                                     f"{n_days}_days_{ts_to_date(time())[:19].replace(':', '')}", ''))
    iters = 10
    if 'iters' in backtest_config:
        iters = backtest_config['iters']
    else:
        print('Parameter iters should be defined in the configuration. Defaulting to 10.')
    num_cpus = 2
    if 'num_cpus' in backtest_config:
        num_cpus = backtest_config['num_cpus']
    else:
        print('Parameter num_cpus should be defined in the configuration. Defaulting to 2.')
    n_particles = 10
    if 'n_particles' in backtest_config:
        n_particles = backtest_config['n_particles']
    phi1 = 1.4962
    phi2 = 1.4962
    omega = 0.7298
    if 'options' in backtest_config:
        phi1 = backtest_config['options']['c1']
        phi2 = backtest_config['options']['c2']
        omega = backtest_config['options']['w']
    current_best_params = []
    if current_best:
        if type(current_best) == list:
            for c in current_best:
                c = clean_start_config(c, config, backtest_config['ranges'])
                current_best_params.append(c)
        else:
            current_best = clean_start_config(current_best, config, backtest_config['ranges'])
            current_best_params.append(current_best)

    ray.init(num_cpus=num_cpus, logging_level=logging.FATAL, log_to_driver=False)
    pso = ng.optimizers.ConfiguredPSO(transform='identity', popsize=n_particles, omega=omega, phip=phi1, phig=phi2)
    algo = NevergradSearch(optimizer=pso, points_to_evaluate=current_best_params)
    algo = ConcurrencyLimiter(algo, max_concurrent=num_cpus)
    scheduler = AsyncHyperBandScheduler()

    analysis = tune.run(tune.with_parameters(wrap_backtest, ticks=ticks), metric='objective', mode='max', name='search',
                        search_alg=algo, scheduler=scheduler, num_samples=iters, config=config, verbose=1,
                        reuse_actors=True, local_dir=session_dirpath,
                        progress_reporter=LogReporter(metric_columns=['daily_gain',
                                                                      'closest_liquidation',
                                                                      'max_hours_between_fills',
                                                                      'objective'],
                                                      parameter_columns=[k for k in backtest_config['ranges'] if type(
                                                          config[k]) == ray.tune.sample.Float or type(
                                                          config[k]) == ray.tune.sample.Integer]))

    ray.shutdown()
    return analysis
Exemplo n.º 26
0
def make_trainable(*, num_epochs, gpus_per_trial, dataset, init_config,
                   init_state_dict, processor):
    return tune.with_parameters(
        clip_fine_tune,
        num_epochs=num_epochs,
        num_gpus=gpus_per_trial,
        dataset=dataset,
        init_config=init_config,
        init_state_dict=init_state_dict,
        processor=processor,
    )
Exemplo n.º 27
0
def main(distributed: bool,
         num_samples: int = 5,
         batch_size: int = 512,
         num_epochs: int = 10) -> None:
    init_logging("main.log")
    logger.info("Running main ...")
    if distributed:
        ray.init(address="localhost:6379",
                 _redis_password=os.getenv("RAY_REDIS_PWD"),
                 ignore_reinit_error=True)
    else:
        ray.init(ignore_reinit_error=True)

    X, y = make_data(NUM_ROW)
    X_tr, X_val, y_tr, y_val = train_test_split(X, y, test_size=0.2)

    # NOTE: Hyperopt config
    metric = "loss"
    mode = "min"
    hp_search = HyperOptSearch(metric=metric, mode=mode)

    # NOTE: Like functools.partial, but stores data in object store
    objective = tune.with_parameters(fit,
                                     X_tr=X_tr,
                                     X_val=X_val,
                                     y_tr=y_tr,
                                     y_val=y_val,
                                     batch_size=batch_size,
                                     num_epochs=num_epochs)

    # NOTE: Define the support of the parameters we're optimizing over
    param_space = {
        "width": tune.choice((2**np.arange(5, 11)).astype(int)),
        "depth": tune.choice(range(1, 5)),
        "lr": tune.loguniform(1e-4, 5e-2)
    }

    logger.info("Starting hyperparameter search ...")
    analysis = tune.run(objective,
                        num_samples=num_samples,
                        config=param_space,
                        search_alg=hp_search,
                        resources_per_trial={
                            "cpu": 2,
                            "gpu": 0.5
                        },
                        metric=metric,
                        mode=mode)
    best_config = analysis.get_best_config(metric=metric, mode=mode)
    logger.info("Best config:\n%s", best_config)
    with open("/tmp/analysis.p", "wb") as f:
        pickle.dump(analysis, f)
    logger.info("Best results %s", pformat(analysis.results))
    analysis.results_df.to_parquet(RESULTS_PATH)
Exemplo n.º 28
0
    def testWithParameters2(self):
        class Data:
            def __init__(self):
                import numpy as np
                self.data = np.random.rand((2 * 1024 * 1024))

        def train(config, data=None):
            tune.report(metric=len(data.data))

        trainable = tune.with_parameters(train, data=Data())
        dumped = cloudpickle.dumps(trainable)
        assert sys.getsizeof(dumped) < 100 * 1024
Exemplo n.º 29
0
def tune_mnist(
    num_samples=10,
    num_epochs=10,
    gpus_per_trial=0,
    tracking_uri=None,
    experiment_name="ptl_autologging_example",
):
    data_dir = os.path.join(tempfile.gettempdir(), "mnist_data_")
    # Download data
    MNISTDataModule(data_dir=data_dir).prepare_data()

    # Set the MLflow experiment, or create it if it does not exist.
    mlflow.set_tracking_uri(tracking_uri)
    mlflow.set_experiment(experiment_name)

    config = {
        "layer_1": tune.choice([32, 64, 128]),
        "layer_2": tune.choice([64, 128, 256]),
        "lr": tune.loguniform(1e-4, 1e-1),
        "batch_size": tune.choice([32, 64, 128]),
        "mlflow": {
            "experiment_name": experiment_name,
            "tracking_uri": mlflow.get_tracking_uri(),
        },
        "data_dir": os.path.join(tempfile.gettempdir(), "mnist_data_"),
        "num_epochs": num_epochs,
    }

    trainable = tune.with_parameters(
        train_mnist_tune,
        data_dir=data_dir,
        num_epochs=num_epochs,
        num_gpus=gpus_per_trial,
    )

    analysis = tune.run(
        trainable,
        resources_per_trial={
            "cpu": 1,
            "gpu": gpus_per_trial
        },
        metric="loss",
        mode="min",
        config=config,
        num_samples=num_samples,
        name="tune_mnist",
    )

    print("Best hyperparameters found were: ", analysis.best_config)
Exemplo n.º 30
0
def main():
    logging.basicConfig(level=logging.INFO)

    # Raylib parameters
    num_samples = 10
    envname = 'AdversarialAntBulletEnv-v0'
    trainingconfig = Path.cwd() / 'trainingconfig.json'
    evaluate_mean_n = 1000  # Number of timesteps over which to evaluate the mean reward
    name_fmt = 'million-bucks_{adv_force}'

    config = {
        # TODO: sample from control once, then different adversarial strengths
        # Range is centered on the force that achieves the closest reward to the control (7.5)
        "adv_force": tune.qrandn(7.5, 2.5, 0.1),
    }

    # https://docs.ray.io/en/master/tune/tutorials/overview.html#which-search-algorithm-scheduler-should-i-choose
    # Use BOHB for larger problems with a small number of hyperparameters
    # search = TuneBOHB(max_concurrent=4, metric="mean_loss", mode="min")
    # sched = HyperBandForBOHB(
    #     time_attr="training_iteration",
    #     max_t=100,
    # )

    # Implicitly use random search if search algo is not specified
    sched = ASHAScheduler(
        time_attr='training_iteration',
        max_t=100,
        grace_period=1,  # Unit is iterations, not timesteps. TODO configure
    )

    # Pass in a Trainable class or function to tune.run.
    local_dir = str(Path.cwd() / "ray")
    logging.info(f'{local_dir=}')
    anal = tune.run(tune.with_parameters(trainable,
                                         envname=envname,
                                         trainingconfig=trainingconfig,
                                         evaluate_mean_n=evaluate_mean_n,
                                         name_fmt=name_fmt),
                    config=config,
                    num_samples=num_samples,
                    scheduler=sched,
                    local_dir=local_dir,
                    metric="robustness",
                    mode="max",
                    log_to_file=True)
    logging.info(f'best config: {anal.best_config}')
    logging.info(f'best config: {anal.best_result}')