def test_fixed_trial_suggest_int() -> None: trial = FixedTrial({"x": 1}) assert trial.suggest_int("x", 0, 10) == 1 with pytest.raises(ValueError): trial.suggest_int("y", 0, 10)
def test_suggest_loguniform() -> None: trial = FixedTrial({"x": 0.99}) assert trial.suggest_loguniform("x", 0.1, 1.0) == 0.99 with pytest.raises(ValueError): trial.suggest_loguniform("y", 0.0, 1.0)
def test_fixed_trial_suggest_discrete_uniform() -> None: trial = FixedTrial({"x": 0.9}) assert trial.suggest_discrete_uniform("x", 0.0, 1.0, 0.1) == 0.9 with pytest.raises(ValueError): trial.suggest_discrete_uniform("y", 0.0, 1.0, 0.1)
def test_fixed_trial_suggest_uniform() -> None: trial = FixedTrial({"x": 1.0}) assert trial.suggest_uniform("x", -100.0, 100.0) == 1.0 with pytest.raises(ValueError): trial.suggest_uniform("y", -100.0, 100.0)
def test_fixed_trial_report(): # type: () -> None # FixedTrial ignores reported values. trial = FixedTrial({}) trial.report(1.0, 1) trial.report(2.0, 2)
def test_fixed_trial_suggest_int_log(): # type: () -> None trial = FixedTrial({"x": 1}) assert trial.suggest_int("x", 1, 10, log=True) == 1 with pytest.raises(ValueError): trial.suggest_int("y", 1, 10, log=True)
def test_fixed_trial_suggest_uniform(): # type: () -> None trial = FixedTrial({'x': 1.}) assert trial.suggest_uniform('x', -100., 100.) == 1. with pytest.raises(ValueError): trial.suggest_uniform('y', -100., 100.)
def test_fixed_trial_params() -> None: params = {"x": 1} trial = FixedTrial(params) assert trial.params == {} assert trial.suggest_uniform("x", 0, 10) == 1 assert trial.params == params
def test_fixed_trial_suggest_int(): # type: () -> None trial = FixedTrial({'x': 1}) assert trial.suggest_int('x', 0, 10) == 1 with pytest.raises(ValueError): trial.suggest_int('y', 0, 10)
def test_fixed_trial_suggest_discrete_uniform(): # type: () -> None trial = FixedTrial({'x': 0.9}) assert trial.suggest_discrete_uniform('x', 0., 1., 0.1) == 0.9 with pytest.raises(ValueError): trial.suggest_discrete_uniform('y', 0., 1., 0.1)
def test_fixed_trial_number() -> None: params = {"x": 1} trial = FixedTrial(params, 2) assert trial.number == 2 trial = FixedTrial(params) assert trial.number == 0
def test_fixed_trial_suggest_categorical(): # type: () -> None trial = FixedTrial({'x': 1}) assert trial.suggest_categorical('x', [0, 1, 2, 3]) == 1 with pytest.raises(ValueError): trial.suggest_categorical('y', [0, 1, 2, 3])
def test_fixed_trial_params(): # type: () -> None params = {'x': 1} trial = FixedTrial(params) assert trial.params == {} assert trial.suggest_uniform('x', 0, 10) == 1 assert trial.params == params
def _create_trial( trial_type: type, params: Optional[Dict[str, Any]] = None, distributions: Optional[Dict[str, BaseDistribution]] = None, ) -> BaseTrial: if params is None: params = {"x": 10} assert params is not None if distributions is None: distributions = {"x": FloatDistribution(5, 12)} assert distributions is not None if trial_type == FixedTrial: return FixedTrial(params) elif trial_type == FrozenTrial: trial = create_trial(value=0.2, params=params, distributions=distributions) trial.number = 0 return trial elif trial_type == Trial: study = create_study() study.enqueue_trial(params) return study.ask() else: assert False
def model_fn(model_dir): device = torch.device("cuda" if torch.cuda.is_available() else "cpu") params = torch.load(os.path.join(model_dir, 'params.pth')) model = define_model(FixedTrial(params)).to(device) with open(os.path.join(model_dir, 'model.pth'), 'rb') as f: model.load_state_dict(torch.load(f)) return model.to(device)
def model_fn(model_dir): """ This function is called by the Chainer container during hosting when running on SageMaker with values populated by the hosting environment. This function loads models written during training into `model_dir`. Args: model_dir (str): path to the directory containing the saved model artifacts Returns: a loaded Chainer model For more on `model_fn`, please visit the sagemaker-python-sdk repository: https://github.com/aws/sagemaker-python-sdk For more on the Chainer container, please visit the sagemaker-chainer-containers repository: https://github.com/aws/sagemaker-chainer-containers """ from optuna.trial import FixedTrial chainer.config.train = False params = np.load(os.path.join(model_dir, 'params.npz'))['arr_0'].item() model = L.Classifier(create_model(FixedTrial(params))) serializers.load_npz(os.path.join(model_dir, 'model.npz'), model) return model.predictor
def test_fixed_trial_suggest_categorical(): # type: () -> None # Integer categories. trial = FixedTrial({'x': 1}) assert trial.suggest_categorical('x', [0, 1, 2, 3]) == 1 with pytest.raises(ValueError): trial.suggest_categorical('y', [0, 1, 2, 3]) # String categories. trial = FixedTrial({'x': 'baz'}) assert trial.suggest_categorical('x', ['foo', 'bar', 'baz']) == 'baz' with pytest.raises(ValueError): trial.suggest_categorical('y', ['foo', 'bar', 'baz'])
def example(): """ In this example we search for best hyperparameters for a Support Vector Machine Classifier on the breast cancer dataset. We split the data for training and testing (we do not need a separate validation set, because POCStudy internally does a cross validation). We run optimization for 60 seconds. You may see some ConvergenceWarning from SVC. Don't worry - those trials are usually pruned anyway. Number of explored points depends on how fast is your machine. We use the best params to train the model on the whole train set and score it on the test set. Finally we plot the optimization history. For first 10 trials, optuna will perform random search to warm up TPE, then it will focus on exploring most promising areas. Observe that after 15th iteration, multiple trials are pruned. """ # Remove next 3 lines if you want to see some SVC warnings. import warnings from sklearn.exceptions import ConvergenceWarning warnings.filterwarnings(action='ignore', category=ConvergenceWarning) # Load and optionally split data. x, y = load_breast_cancer(return_X_y=True) dataset = Dataset(x, y, 'sklearn_breast_cancer') splitter = StratifiedSplitter(train_ratio=0.7, val_ratio=0.0, random_state=2020) split = splitter.split(dataset) # Instantiate objective with train data. # You can pass a list of scorers - the first one wll be used for # optimization and the rest just for monitoring objective = MultiLevelSVMObjective( split.train, scorer=['accuracy', 'f1', 'recall', 'precision']) # Create study and run optimization (internally it will use 5-fold CV) study = POCStudy(objective) study.optimize(timeout_seconds=60, num_trials=None) # Now you can use best trial params to create a model... best_params = study.best_trial.params print('Creating model with best params:', best_params) best_trial = FixedTrial(best_params) best_model = objective.create_model(best_trial) # ... which you can fit and score. best_model.fit(X=split.train.x, y=split.train.y) test_score = best_model.score(X=split.test.x, y=split.test.y) print(f'test_score: {test_score:.3g}') # Plot optimization history. print('Plot will hopefully open in a browser.') fig = optuna.visualization.plot_optimization_history(study) fig.show() print('Done.')
def best_models(self, return_preds=False): fixed_trial = FixedTrial(self.study.best_params) loss, models, cv_preds = self.objective(fixed_trial, return_model=True) if return_preds: return models, cv_preds return models
def test_suggest_float() -> None: trial = FixedTrial({"x": 1.0}) assert trial.suggest_float("x", -100.0, 100.0) == 1.0 with pytest.raises(ValueError): trial.suggest_float("x", -100, 100, step=10, log=True) with pytest.raises(ValueError): trial.suggest_uniform("y", -100.0, 100.0)
def test_suggest_int_log() -> None: trial = FixedTrial({"x": 1}) assert trial.suggest_int("x", 1, 10, log=True) == 1 with pytest.raises(ValueError): trial.suggest_int("x", 1, 10, step=2, log=True) with pytest.raises(ValueError): trial.suggest_int("y", 1, 10, log=True)
def test_fixed_trial_user_attrs(): # type: () -> None trial = FixedTrial({'x': 1}) trial.set_user_attr('data', 'MNIST') assert trial.user_attrs['data'] == 'MNIST'
def test_fixed_trial_system_attrs(): # type: () -> None trial = FixedTrial({'x': 1}) trial.set_system_attr('system_message', 'test') assert trial.system_attrs['system_message'] == 'test'
def test_fixed_trial_datetime_start(): # type: () -> None params = {'x': 1} trial = FixedTrial(params) assert trial.datetime_start is not None
def test_fixed_trial_user_attrs() -> None: trial = FixedTrial({"x": 1}) trial.set_user_attr("data", "MNIST") assert trial.user_attrs["data"] == "MNIST"
def test_fixed_trial_system_attrs() -> None: trial = FixedTrial({"x": 1}) trial.set_system_attr("system_message", "test") assert trial.system_attrs["system_message"] == "test"
def test_fixed_trial_should_prune() -> None: # FixedTrial never prunes trials. assert FixedTrial({}).should_prune() is False
def test_fixed_trial_datetime_start() -> None: params = {"x": 1} trial = FixedTrial(params) assert trial.datetime_start is not None
def test_fixed_trial_should_prune(): # type: () -> None # FixedTrial never prunes trials. assert FixedTrial({}).should_prune() is False assert FixedTrial({}).should_prune(1) is False
def test_fixed_trial_suggest_categorical() -> None: # Integer categories. trial = FixedTrial({"x": 1}) assert trial.suggest_categorical("x", [0, 1, 2, 3]) == 1 with pytest.raises(ValueError): trial.suggest_categorical("y", [0, 1, 2, 3]) # String categories. trial = FixedTrial({"x": "baz"}) assert trial.suggest_categorical("x", ["foo", "bar", "baz"]) == "baz" # Unknown parameter. with pytest.raises(ValueError): trial.suggest_categorical("y", ["foo", "bar", "baz"]) # Not in choices. with pytest.raises(ValueError): trial.suggest_categorical("x", ["foo", "bar"]) # Unknown parameter and bad category type. with pytest.warns(UserWarning): with pytest.raises(ValueError): # Must come after `pytest.warns` to catch failures. trial.suggest_categorical("x", [{"foo": "bar"}]) # type: ignore