def _hp_search(self, input_df, validation_df, metric, recipe, mc,
resources_per_trial, remote_dir):
ft = self.create_feature_transformer()
feature_list = ft.get_feature_list()
model_fn = partial(self.create_model,
resources_per_trial=resources_per_trial)
# prepare parameters for search engine
search_space = recipe.search_space(feature_list)
searcher = RayTuneSearchEngine(
logs_dir=self.logs_dir,
resources_per_trial=resources_per_trial,
name=self.name,
remote_dir=remote_dir,
)
searcher.compile(
input_df,
model_create_func=model_fn,
search_space=search_space,
recipe=recipe,
feature_transformers=ft,
validation_df=validation_df,
metric=metric,
mc=mc,
)
# searcher.test_run()
analysis = searcher.run()
pipeline = self._make_pipeline(analysis,
feature_transformers=ft,
model_create_fn=model_fn,
remote_dir=remote_dir)
return pipeline
def _hp_search(self,
input_df,
validation_df,
metric,
recipe,
mc,
resources_per_trial,
remote_dir):
ft = self.create_feature_transformer()
model_fn = self.make_model_fn(resources_per_trial)
self._detach_recipe(recipe)
# prepare parameters for search engine
searcher = RayTuneSearchEngine(logs_dir=self.logs_dir,
resources_per_trial=resources_per_trial,
name=self.name,
remote_dir=remote_dir,
)
searcher.compile(data=input_df,
model_create_func=model_fn,
validation_data=validation_df,
search_space=self.search_space,
n_sampling=self.num_samples,
epochs=self.epochs,
metric_threshold=self.metric_threshold,
search_alg=self.search_alg,
search_alg_params=self.search_alg_params,
scheduler=self.scheduler,
scheduler_params=self.scheduler_params,
feature_transformers=ft,
metric=metric,
mc=mc,
)
# searcher.test_run()
analysis = searcher.run()
pipeline = self._make_pipeline(analysis,
feature_transformers=ft,
model=model_fn(),
remote_dir=remote_dir)
return pipeline
def _hp_search(self, input_df, validation_df, metric, recipe, mc,
resources_per_trial, remote_dir):
ft = self.create_feature_transformer()
try:
feature_list = ft.get_feature_list()
except:
feature_list = None
model_fn = self.make_model_fn(resources_per_trial)
# prepare parameters for search engine
search_space = recipe.search_space(feature_list)
searcher = RayTuneSearchEngine(
logs_dir=self.logs_dir,
resources_per_trial=resources_per_trial,
name=self.name,
remote_dir=remote_dir,
)
searcher.compile(
data={
'df': input_df,
'val_df': validation_df
},
model_create_func=model_fn,
search_space=search_space,
recipe=recipe,
search_alg=self.search_alg,
search_alg_params=self.search_alg_params,
scheduler=self.scheduler,
scheduler_params=self.scheduler_params,
feature_transformers=ft,
metric=metric,
mc=mc,
)
# searcher.test_run()
analysis = searcher.run()
pipeline = self._make_pipeline(analysis,
feature_transformers=ft,
model=model_fn(),
remote_dir=remote_dir)
return pipeline
def test_searcher_metric(self):
train_x, train_y, val_x, val_y = get_np_input()
data = (train_x, train_y)
val_data = (val_x, val_y)
# test metric name is returned and max mode can be stopped
searcher = prepare_searcher(data=data,
validation_data=val_data,
name='test_searcher_metric_name',
metric='mse',
search_space=create_simple_search_space(),
stop=create_stop(float('inf')))
analysis = searcher.run()
sorted_results = list(
map(
lambda x: x.last_result['mse'],
RayTuneSearchEngine._get_sorted_trials(analysis.trials,
metric='mse',
mode="min")))
# assert metric name is reported
assert 'mse' in analysis.trials[0].last_result.keys()
# assert _get_sorted_trials get increasing result
assert all(sorted_results[i] <= sorted_results[i + 1]
for i in range(len(sorted_results) - 1))
# assert _get_best_result get minimum result
assert RayTuneSearchEngine._get_best_result(
analysis.trials, metric='mse',
mode="min")['mse'] == sorted_results[0]
assert all(analysis.trials[i].last_result['mse'] >=
analysis.trials[i].last_result['best_mse']
for i in range(len(sorted_results)))
# assert the trail stop at once since mse has mode of 'min'
assert analysis.trials[0].last_result['iterations_since_restore'] == 1
# max mode metric with stop
searcher = prepare_searcher(data=data,
validation_data=val_data,
name='test_searcher_metric_name',
metric='r2',
search_space=create_simple_search_space(),
stop=create_stop(0)) # stop at once
analysis = searcher.run()
sorted_results = list(
map(
lambda x: x.last_result['r2'],
RayTuneSearchEngine._get_sorted_trials(analysis.trials,
metric='r2',
mode="max")))
# assert metric name is reported
assert 'r2' in analysis.trials[0].last_result.keys()
# assert _get_sorted_trials get decreasing result
assert all(sorted_results[i] >= sorted_results[i + 1]
for i in range(len(sorted_results) - 1))
# assert _get_best_result get maximum result
assert RayTuneSearchEngine._get_best_result(
analysis.trials, metric='r2',
mode="max")['r2'] == sorted_results[0]
assert all(analysis.trials[i].last_result['r2'] <=
analysis.trials[i].last_result['best_r2']
for i in range(len(sorted_results)))
# assert the trail stop at once since mse has mode of 'max'
assert analysis.trials[0].last_result['iterations_since_restore'] == 1
# test min mode metric without stop
searcher = prepare_searcher(
data=data,
validation_data=val_data,
name='test_searcher_metric_name',
metric='mae',
search_space=create_simple_search_space(),
stop=create_stop(0)) # never stop by metric
analysis = searcher.run()
sorted_results = list(
map(
lambda x: x.last_result['mae'],
RayTuneSearchEngine._get_sorted_trials(analysis.trials,
metric='mae',
mode="min")))
# assert metric name is reported
assert 'mae' in analysis.trials[0].last_result.keys()
# assert _get_sorted_trials get increasing result
assert all(sorted_results[i] <= sorted_results[i + 1]
for i in range(len(sorted_results) - 1))
# assert _get_best_result get minimum result
assert RayTuneSearchEngine._get_best_result(
analysis.trials, metric='mae',
mode="min")['mae'] == sorted_results[0]
assert all(analysis.trials[i].last_result['mae'] >=
analysis.trials[i].last_result['best_mae']
for i in range(len(sorted_results)))
# assert the trail stop at once since mse has mode of 'min'
assert analysis.trials[0].last_result['iterations_since_restore'] == 20