def _get_sync_client_and_remote_checkpoint_dir(
self,
trial_dir: Path) -> Optional[Tuple["CommandBasedClient", str]]:
"""Get the Ray sync client and path to remote checkpoint directory."""
if self.sync_config is None:
return None
remote_checkpoint_dir = os.path.join(
self.sync_config.upload_dir,
*_get_relative_checkpoints_dir_parts(trial_dir))
return get_cloud_sync_client(
remote_checkpoint_dir), remote_checkpoint_dir
def testCloudSyncExclude(self):
captured = deque(maxlen=1)
captured.append("")
def always_true(*args, **kwargs):
return True
def capture_popen(command, *args, **kwargs):
captured.append(command)
with patch("subprocess.Popen",
capture_popen), patch("distutils.spawn.find_executable",
always_true):
# S3
s3_client = get_cloud_sync_client("s3://test-bucket/test-dir")
s3_client.sync_down("s3://test-bucket/test-dir/remote_source",
"local_target")
self.assertEqual(
captured[0].strip(),
"aws s3 sync s3://test-bucket/test-dir/remote_source "
"local_target --only-show-errors",
)
s3_client.sync_down(
"s3://test-bucket/test-dir/remote_source",
"local_target",
exclude=["*/checkpoint_*"],
)
self.assertEqual(
captured[0].strip(),
"aws s3 sync s3://test-bucket/test-dir/remote_source "
"local_target --only-show-errors "
"--exclude '*/checkpoint_*'",
)
s3_client.sync_down(
"s3://test-bucket/test-dir/remote_source",
"local_target",
exclude=["*/checkpoint_*", "*.big"],
)
self.assertEqual(
captured[0].strip(),
"aws s3 sync s3://test-bucket/test-dir/remote_source "
"local_target --only-show-errors "
"--exclude '*/checkpoint_*' --exclude '*.big'",
)
# GS
gs_client = get_cloud_sync_client("gs://test-bucket/test-dir")
gs_client.sync_down("gs://test-bucket/test-dir/remote_source",
"local_target")
self.assertEqual(
captured[0].strip(),
"gsutil rsync -r "
"gs://test-bucket/test-dir/remote_source "
"local_target",
)
gs_client.sync_down(
"gs://test-bucket/test-dir/remote_source",
"local_target",
exclude=["*/checkpoint_*"],
)
self.assertEqual(
captured[0].strip(),
"gsutil rsync -r "
"-x '(.*/checkpoint_.*)' "
"gs://test-bucket/test-dir/remote_source "
"local_target",
)
gs_client.sync_down(
"gs://test-bucket/test-dir/remote_source",
"local_target",
exclude=["*/checkpoint_*", "*.big"],
)
self.assertEqual(
captured[0].strip(),
"gsutil rsync -r "
"-x '(.*/checkpoint_.*)|(.*.big)' "
"gs://test-bucket/test-dir/remote_source "
"local_target",
)
def _create_storage_client(self):
"""Returns a storage client."""
return get_cloud_sync_client(self.remote_checkpoint_dir)
def execute(
self,
config,
dataset=None,
training_set=None,
validation_set=None,
test_set=None,
training_set_metadata=None,
data_format=None,
experiment_name="hyperopt",
model_name="run",
resume=None,
skip_save_training_description=False,
skip_save_training_statistics=False,
skip_save_model=False,
skip_save_progress=False,
skip_save_log=False,
skip_save_processed_input=True,
skip_save_unprocessed_output=False,
skip_save_predictions=False,
skip_save_eval_stats=False,
output_directory="results",
gpus=None,
gpu_memory_limit=None,
allow_parallel_threads=True,
callbacks=None,
backend=None,
random_seed=default_random_seed,
debug=False,
hyperopt_log_verbosity=3,
features_eligible_for_shared_params=None,
**kwargs,
) -> RayTuneResults:
if isinstance(dataset, str) and not has_remote_protocol(
dataset) and not os.path.isabs(dataset):
dataset = os.path.abspath(dataset)
if isinstance(backend, str):
backend = initialize_backend(backend)
if gpus is not None:
raise ValueError(
"Parameter `gpus` is not supported when using Ray Tune. "
"Configure GPU resources with Ray and set `gpu_resources_per_trial` in your "
"hyperopt config.")
if gpu_memory_limit is None and 0 < self._gpu_resources_per_trial_non_none < 1:
# Enforce fractional GPU utilization
gpu_memory_limit = self.gpu_resources_per_trial
hyperopt_dict = dict(
config=config,
dataset=dataset,
training_set=training_set,
validation_set=validation_set,
test_set=test_set,
training_set_metadata=training_set_metadata,
data_format=data_format,
experiment_name=experiment_name,
model_name=model_name,
eval_split=self.split,
skip_save_training_description=skip_save_training_description,
skip_save_training_statistics=skip_save_training_statistics,
skip_save_model=skip_save_model,
skip_save_progress=skip_save_progress,
skip_save_log=skip_save_log,
skip_save_processed_input=skip_save_processed_input,
skip_save_unprocessed_output=skip_save_unprocessed_output,
skip_save_predictions=skip_save_predictions,
skip_save_eval_stats=skip_save_eval_stats,
output_directory=output_directory,
gpus=gpus,
gpu_memory_limit=gpu_memory_limit,
allow_parallel_threads=allow_parallel_threads,
callbacks=callbacks,
backend=backend,
random_seed=random_seed,
debug=debug,
)
mode = "min" if self.goal != MAXIMIZE else "max"
metric = "metric_score"
# if random seed not set, use Ludwig seed
self.search_algorithm.check_for_random_seed(random_seed)
if self.search_algorithm.search_alg_dict is not None:
if TYPE not in self.search_algorithm.search_alg_dict:
candiate_search_algs = [
search_alg for search_alg in SEARCH_ALG_IMPORT.keys()
]
logger.warning(
"WARNING: search_alg type parameter missing, using 'variant_generator' as default. "
f"These are possible values for the type parameter: {candiate_search_algs}."
)
search_alg = None
else:
search_alg_type = self.search_algorithm.search_alg_dict[TYPE]
search_alg = tune.create_searcher(
search_alg_type,
metric=metric,
mode=mode,
**self.search_algorithm.search_alg_dict)
else:
search_alg = None
if self.max_concurrent_trials:
assert (
self.max_concurrent_trials > 0
), f"`max_concurrent_trials` must be greater than 0, got {self.max_concurrent_trials}"
if isinstance(search_alg,
BasicVariantGenerator) or search_alg is None:
search_alg = BasicVariantGenerator(
max_concurrent=self.max_concurrent_trials)
elif isinstance(search_alg, ConcurrencyLimiter):
raise ValueError(
"You have specified `max_concurrent_trials`, but the search "
"algorithm is already a `ConcurrencyLimiter`. FIX THIS "
"by setting `max_concurrent_trials=None`.")
else:
search_alg = ConcurrencyLimiter(
search_alg, max_concurrent=self.max_concurrent_trials)
resources_per_trial = {
"cpu": self._cpu_resources_per_trial_non_none,
"gpu": self._gpu_resources_per_trial_non_none,
}
def run_experiment_trial(config,
local_hyperopt_dict,
checkpoint_dir=None):
return self._run_experiment(
config,
checkpoint_dir,
local_hyperopt_dict,
self.decode_ctx,
features_eligible_for_shared_params,
_is_ray_backend(backend),
)
tune_config = {}
tune_callbacks = []
for callback in callbacks or []:
run_experiment_trial, tune_config = callback.prepare_ray_tune(
run_experiment_trial,
tune_config,
tune_callbacks,
)
if _is_ray_backend(backend):
# for now, we do not do distributed training on cpu (until spread scheduling is implemented for Ray Train)
# but we do want to enable it when GPUs are specified
resources_per_trial = PlacementGroupFactory(
[{}] + ([{
"CPU": 0,
"GPU": 1
}] * self._gpu_resources_per_trial_non_none) if self.
_gpu_resources_per_trial_non_none else [{}] +
[{
"CPU": self._cpu_resources_per_trial_non_none
}])
if has_remote_protocol(output_directory):
run_experiment_trial = tune.durable(run_experiment_trial)
self.sync_config = tune.SyncConfig(sync_to_driver=False,
upload_dir=output_directory)
if _ray_114:
self.sync_client = get_node_to_storage_syncer(
SyncConfig(upload_dir=output_directory))
else:
self.sync_client = get_cloud_sync_client(output_directory)
output_directory = None
elif self.kubernetes_namespace:
from ray.tune.integration.kubernetes import KubernetesSyncClient, NamespacedKubernetesSyncer
self.sync_config = tune.SyncConfig(
sync_to_driver=NamespacedKubernetesSyncer(
self.kubernetes_namespace))
self.sync_client = KubernetesSyncClient(self.kubernetes_namespace)
run_experiment_trial_params = tune.with_parameters(
run_experiment_trial, local_hyperopt_dict=hyperopt_dict)
register_trainable(
f"trainable_func_f{hash_dict(config).decode('ascii')}",
run_experiment_trial_params)
# Note that resume="AUTO" will attempt to resume the experiment if possible, and
# otherwise will start a new experiment:
# https://docs.ray.io/en/latest/tune/tutorials/tune-stopping.html
should_resume = "AUTO" if resume is None else resume
try:
analysis = tune.run(
f"trainable_func_f{hash_dict(config).decode('ascii')}",
name=experiment_name,
config={
**self.search_space,
**tune_config,
},
scheduler=self.scheduler,
search_alg=search_alg,
num_samples=self.num_samples,
keep_checkpoints_num=1,
max_failures=1, # retry a trial failure once
resources_per_trial=resources_per_trial,
time_budget_s=self.time_budget_s,
sync_config=self.sync_config,
local_dir=output_directory,
metric=metric,
mode=mode,
trial_name_creator=lambda trial: f"trial_{trial.trial_id}",
trial_dirname_creator=lambda trial: f"trial_{trial.trial_id}",
callbacks=tune_callbacks,
stop=CallbackStopper(callbacks),
verbose=hyperopt_log_verbosity,
resume=should_resume,
log_to_file=True,
)
except Exception as e:
# Explicitly raise a RuntimeError if an error is encountered during a Ray trial.
# NOTE: Cascading the exception with "raise _ from e" still results in hanging.
raise RuntimeError(f"Encountered Ray Tune error: {e}")
if "metric_score" in analysis.results_df.columns:
ordered_trials = analysis.results_df.sort_values(
"metric_score", ascending=self.goal != MAXIMIZE)
# Catch nans in edge case where the trial doesn't complete
temp_ordered_trials = []
for kwargs in ordered_trials.to_dict(orient="records"):
for key in ["parameters", "training_stats", "eval_stats"]:
if isinstance(kwargs[key], float):
kwargs[key] = {}
temp_ordered_trials.append(kwargs)
# Trials w/empty eval_stats fields & non-empty training_stats fields ran intermediate
# tune.report call(s) but were terminated before reporting eval_stats from post-train
# evaluation (e.g., trial stopped due to time budget or relatively poor performance.)
# For any such trials, run model evaluation for the best model in that trial & record
# results in ordered_trials which is returned & is persisted in hyperopt_statistics.json.
for trial in temp_ordered_trials:
if trial["eval_stats"] == "{}" and trial[
"training_stats"] != "{}":
# Evaluate the best model on the eval_split, which is validation_set
if validation_set is not None and validation_set.size > 0:
trial_path = trial["trial_dir"]
best_model_path = self._get_best_model_path(
trial_path, analysis)
if best_model_path is not None:
self._evaluate_best_model(
trial,
trial_path,
best_model_path,
validation_set,
data_format,
skip_save_unprocessed_output,
skip_save_predictions,
skip_save_eval_stats,
gpus,
gpu_memory_limit,
allow_parallel_threads,
backend,
debug,
)
else:
logger.warning(
"Skipping evaluation as no model checkpoints were available"
)
else:
logger.warning(
"Skipping evaluation as no validation set was provided"
)
ordered_trials = [
TrialResults.from_dict(load_json_values(kwargs))
for kwargs in temp_ordered_trials
]
else:
logger.warning(
"No trials reported results; check if time budget lower than epoch latency"
)
ordered_trials = []
return RayTuneResults(ordered_trials=ordered_trials,
experiment_analysis=analysis)
