def test_syncer_callback_no_size_limit(temp_data_dirs):
"""Check if max_size_bytes is set to None for sync function"""
tmp_source, _ = temp_data_dirs
syncer_callback = SyncerCallback()
trial1 = MockTrial(trial_id="a", logdir=tmp_source)
sync_fn = syncer_callback._get_trial_sync_process(trial1)._fn
assert sync_fn.keywords["max_size_bytes"] is None
def test_cluster_down_full(start_connected_cluster, tmpdir, durable):
"""Tests that run_experiment restoring works on cluster shutdown."""
cluster = start_connected_cluster
dirpath = str(tmpdir)
if durable:
upload_dir = "file://" + str(tmpdir)
syncer_callback = SyncerCallback()
else:
upload_dir = None
syncer_callback = custom_driver_logdir_callback(str(tmpdir))
from ray.tune.result import DEFAULT_RESULTS_DIR
local_dir = DEFAULT_RESULTS_DIR
base_dict = dict(
run="__fake",
stop=dict(training_iteration=3),
local_dir=local_dir,
sync_config=dict(upload_dir=upload_dir),
)
exp1_args = base_dict
exp2_args = dict(base_dict.items(), local_dir=dirpath, checkpoint_freq=1)
exp3_args = dict(base_dict.items(), config=dict(mock_error=True))
exp4_args = dict(base_dict.items(),
config=dict(mock_error=True),
checkpoint_freq=1)
all_experiments = {
"exp1": exp1_args,
"exp2": exp2_args,
"exp3": exp3_args,
"exp4": exp4_args,
}
tune.run_experiments(all_experiments,
callbacks=[syncer_callback],
raise_on_failed_trial=False)
ray.shutdown()
cluster.shutdown()
cluster = _start_new_cluster()
trials = tune.run_experiments(
all_experiments,
resume=True,
raise_on_failed_trial=False,
)
assert len(trials) == 4
assert all(t.status in [Trial.TERMINATED, Trial.ERROR] for t in trials)
ray.shutdown()
cluster.shutdown()
def testCallbackReordering(self):
"""SyncerCallback should come after ExperimentLogger callbacks"""
def get_positions(callbacks):
first_logger_pos = None
last_logger_pos = None
syncer_pos = None
for i, callback in enumerate(callbacks):
if isinstance(callback, ExperimentLogger):
if first_logger_pos is None:
first_logger_pos = i
last_logger_pos = i
elif isinstance(callback, SyncerCallback):
syncer_pos = i
return first_logger_pos, last_logger_pos, syncer_pos
# Auto creation of loggers, no callbacks, no syncer
callbacks = create_default_callbacks(None, SyncConfig(), None)
first_logger_pos, last_logger_pos, syncer_pos = get_positions(
callbacks)
self.assertLess(last_logger_pos, syncer_pos)
# Auto creation of loggers with callbacks
callbacks = create_default_callbacks([Callback()], SyncConfig(), None)
first_logger_pos, last_logger_pos, syncer_pos = get_positions(
callbacks)
self.assertLess(last_logger_pos, syncer_pos)
# Auto creation of loggers with existing logger (but no CSV/JSON)
callbacks = create_default_callbacks([ExperimentLogger()],
SyncConfig(), None)
first_logger_pos, last_logger_pos, syncer_pos = get_positions(
callbacks)
self.assertLess(last_logger_pos, syncer_pos)
# This should throw an error as the syncer comes before the logger
with self.assertRaises(ValueError):
callbacks = create_default_callbacks(
[SyncerCallback(None),
ExperimentLogger()], SyncConfig(), None)
# This should be reordered but preserve the regular callback order
[mc1, mc2, mc3] = [Callback(), Callback(), Callback()]
# Has to be legacy logger to avoid logger callback creation
lc = LegacyExperimentLogger(logger_classes=DEFAULT_LOGGERS)
callbacks = create_default_callbacks([mc1, mc2, lc, mc3], SyncConfig(),
None)
print(callbacks)
first_logger_pos, last_logger_pos, syncer_pos = get_positions(
callbacks)
self.assertLess(last_logger_pos, syncer_pos)
self.assertLess(callbacks.index(mc1), callbacks.index(mc2))
self.assertLess(callbacks.index(mc2), callbacks.index(mc3))
self.assertLess(callbacks.index(lc), callbacks.index(mc3))
# Syncer callback is appended
self.assertLess(callbacks.index(mc3), syncer_pos)
def test_migration_checkpoint_removal(start_connected_emptyhead_cluster,
tmpdir, durable):
"""Test checks that trial restarts if checkpoint is lost w/ node fail."""
cluster = start_connected_emptyhead_cluster
node = cluster.add_node(num_cpus=1)
cluster.wait_for_nodes()
if durable:
upload_dir = "file://" + str(tmpdir)
syncer_callback = SyncerCallback()
else:
upload_dir = None
syncer_callback = custom_driver_logdir_callback(str(tmpdir))
runner = TrialRunner(BasicVariantGenerator(), callbacks=[syncer_callback])
kwargs = {
"stopping_criterion": {
"training_iteration": 4
},
"checkpoint_freq": 2,
"max_failures": 2,
"remote_checkpoint_dir": upload_dir,
}
# Test recovery of trial that has been checkpointed
t1 = Trial("__fake", **kwargs)
runner.add_trial(t1)
# Start trial, process result (x2), process save
while not t1.has_checkpoint():
runner.step()
cluster.add_node(num_cpus=1)
cluster.remove_node(node)
cluster.wait_for_nodes()
# Remove checkpoint on "remote" node
shutil.rmtree(os.path.dirname(t1.checkpoint.dir_or_data))
if not durable:
# Recover from driver file
t1.checkpoint.dir_or_data = os.path.join(
tmpdir,
t1.relative_logdir,
os.path.relpath(t1.checkpoint.dir_or_data, t1.logdir),
)
while not runner.is_finished():
runner.step()
assert t1.status == Trial.TERMINATED, runner.debug_string()
def create_default_callbacks(callbacks: Optional[List[Callback]],
sync_config: SyncConfig):
callbacks = callbacks or []
# Check if there is a SyncerCallback
has_syncer_callback = any(isinstance(c, SyncerCallback) for c in callbacks)
# If no SyncerCallback was found, add
if not has_syncer_callback and os.environ.get(
"TUNE_DISABLE_AUTO_CALLBACK_SYNCER", "0") != "1":
syncer_callback = SyncerCallback(
sync_function=sync_config.sync_to_driver)
callbacks.append(syncer_callback)
return callbacks
def test_trial_requeue(start_connected_emptyhead_cluster, tmpdir, durable):
"""Removing a node in full cluster causes Trial to be requeued."""
os.environ["TUNE_MAX_PENDING_TRIALS_PG"] = "1"
cluster = start_connected_emptyhead_cluster
node = cluster.add_node(num_cpus=1)
cluster.wait_for_nodes()
if durable:
upload_dir = "file://" + str(tmpdir)
syncer_callback = SyncerCallback()
else:
upload_dir = None
syncer_callback = custom_driver_logdir_callback(str(tmpdir))
runner = TrialRunner(BasicVariantGenerator(),
callbacks=[syncer_callback]) # noqa
kwargs = {
"stopping_criterion": {
"training_iteration": 5
},
"checkpoint_freq": 1,
"max_failures": 1,
"remote_checkpoint_dir": upload_dir,
}
trials = [Trial("__fake", **kwargs), Trial("__fake", **kwargs)]
for t in trials:
runner.add_trial(t)
runner.step() # Start trial
runner.step() # Process result, dispatch save
runner.step() # Process save
running_trials = _get_running_trials(runner)
assert len(running_trials) == 1
assert _check_trial_running(running_trials[0])
cluster.remove_node(node)
cluster.wait_for_nodes()
time.sleep(0.1) # Sleep so that next step() refreshes cluster resources
runner.step() # Process result, dispatch save
runner.step() # Process save (detect error), requeue trial
assert all(t.status == Trial.PENDING
for t in trials), runner.debug_string()
def create_default_callbacks(callbacks: Optional[List[Callback]],
sync_config: SyncConfig,
loggers: Optional[List[Logger]],
metric: Optional[str] = None):
"""Create default callbacks for `tune.run()`.
This function takes a list of existing callbacks and adds default
callbacks to it.
Specifically, three kinds of callbacks will be added:
1. Loggers. Ray Tune's experiment analysis relies on CSV and JSON logging.
2. Syncer. Ray Tune synchronizes logs and checkpoint between workers and
the head node.
2. Trial progress reporter. For reporting intermediate progress, like trial
results, Ray Tune uses a callback.
These callbacks will only be added if they don't already exist, i.e. if
they haven't been passed (and configured) by the user. A notable case
is when a Logger is passed, which is not a CSV or JSON logger - then
a CSV and JSON logger will still be created.
Lastly, this function will ensure that the Syncer callback comes after all
Logger callbacks, to ensure that the most up-to-date logs and checkpoints
are synced across nodes.
"""
callbacks = callbacks or []
has_syncer_callback = False
has_csv_logger = False
has_json_logger = False
has_tbx_logger = False
has_trial_progress_callback = any(
isinstance(c, TrialProgressCallback) for c in callbacks)
if not has_trial_progress_callback:
trial_progress_callback = TrialProgressCallback(metric=metric)
callbacks.append(trial_progress_callback)
# Track syncer obj/index to move callback after loggers
last_logger_index = None
syncer_index = None
# Create LegacyLoggerCallback for passed Logger classes
if loggers:
# Todo(krfricke): Deprecate `loggers` argument, print warning here.
# Add warning as soon as we ported all loggers to LoggerCallback
# classes.
add_loggers = []
for trial_logger in loggers:
if isinstance(trial_logger, LoggerCallback):
callbacks.append(trial_logger)
elif isinstance(trial_logger, type) and issubclass(
trial_logger, Logger):
add_loggers.append(trial_logger)
else:
raise ValueError(
f"Invalid value passed to `loggers` argument of "
f"`tune.run()`: {trial_logger}")
if add_loggers:
callbacks.append(LegacyLoggerCallback(add_loggers))
# Check if we have a CSV, JSON and TensorboardX logger
for i, callback in enumerate(callbacks):
if isinstance(callback, LegacyLoggerCallback):
last_logger_index = i
if CSVLogger in callback.logger_classes:
has_csv_logger = True
if JsonLogger in callback.logger_classes:
has_json_logger = True
if TBXLogger in callback.logger_classes:
has_tbx_logger = True
elif isinstance(callback, CSVLoggerCallback):
has_csv_logger = True
last_logger_index = i
elif isinstance(callback, JsonLoggerCallback):
has_json_logger = True
last_logger_index = i
elif isinstance(callback, TBXLoggerCallback):
has_tbx_logger = True
last_logger_index = i
elif isinstance(callback, SyncerCallback):
syncer_index = i
has_syncer_callback = True
# If CSV, JSON or TensorboardX loggers are missing, add
if os.environ.get("TUNE_DISABLE_AUTO_CALLBACK_LOGGERS", "0") != "1":
if not has_csv_logger:
callbacks.append(CSVLoggerCallback())
last_logger_index = len(callbacks) - 1
if not has_json_logger:
callbacks.append(JsonLoggerCallback())
last_logger_index = len(callbacks) - 1
if not has_tbx_logger:
try:
callbacks.append(TBXLoggerCallback())
last_logger_index = len(callbacks) - 1
except ImportError:
logger.warning(
"The TensorboardX logger cannot be instantiated because "
"either TensorboardX or one of it's dependencies is not "
"installed. Please make sure you have the latest version "
"of TensorboardX installed: `pip install -U tensorboardx`")
# If no SyncerCallback was found, add
if not has_syncer_callback and os.environ.get(
"TUNE_DISABLE_AUTO_CALLBACK_SYNCER", "0") != "1":
# Detect Docker and Kubernetes environments
_sync_to_driver = detect_sync_to_driver(sync_config.sync_to_driver)
syncer_callback = SyncerCallback(sync_function=_sync_to_driver)
callbacks.append(syncer_callback)
syncer_index = len(callbacks) - 1
if syncer_index is not None and last_logger_index is not None and \
syncer_index < last_logger_index:
if (not has_csv_logger or not has_json_logger or not has_tbx_logger) \
and not loggers:
# Only raise the warning if the loggers were passed by the user.
# (I.e. don't warn if this was automatic behavior and they only
# passed a customer SyncerCallback).
raise ValueError(
"The `SyncerCallback` you passed to `tune.run()` came before "
"at least one `LoggerCallback`. Syncing should be done "
"after writing logs. Please re-order the callbacks so that "
"the `SyncerCallback` comes after any `LoggerCallback`.")
else:
# If these loggers were automatically created. just re-order
# the callbacks
syncer_obj = callbacks[syncer_index]
callbacks.pop(syncer_index)
callbacks.insert(last_logger_index, syncer_obj)
return callbacks
def test_trial_migration(start_connected_emptyhead_cluster, tmpdir, durable):
"""Removing a node while cluster has space should migrate trial.
The trial state should also be consistent with the checkpoint.
"""
cluster = start_connected_emptyhead_cluster
node = cluster.add_node(num_cpus=1)
cluster.wait_for_nodes()
if durable:
upload_dir = "file://" + str(tmpdir)
syncer_callback = SyncerCallback()
else:
upload_dir = None
syncer_callback = custom_driver_logdir_callback(str(tmpdir))
runner = TrialRunner(BasicVariantGenerator(), callbacks=[syncer_callback])
kwargs = {
"stopping_criterion": {
"training_iteration": 4
},
"checkpoint_freq": 2,
"max_failures": 2,
"remote_checkpoint_dir": upload_dir,
}
# Test recovery of trial that hasn't been checkpointed
t = Trial("__fake", **kwargs)
runner.add_trial(t)
runner.step() # Start trial
runner.step() # Process result
assert t.last_result
node2 = cluster.add_node(num_cpus=1)
cluster.remove_node(node)
cluster.wait_for_nodes()
# TODO(ujvl): Node failure does not propagate until a step after it
# actually should. This is possibly a problem with `Cluster`.
runner.step()
runner.step() # Recovery step
# TODO(rliaw): This assertion is not critical but will not pass
# because checkpoint handling is messy and should be refactored
# rather than hotfixed.
# assert t.last_result is None, "Trial result not restored correctly."
# Process result (x2), process save, process result (x2), process save
while not runner.is_finished():
runner.step()
assert t.status == Trial.TERMINATED, runner.debug_string()
# Test recovery of trial that has been checkpointed
t2 = Trial("__fake", **kwargs)
runner.add_trial(t2)
# Start trial, process result (x2), process save
while not t2.has_checkpoint():
runner.step()
node3 = cluster.add_node(num_cpus=1)
cluster.remove_node(node2)
cluster.wait_for_nodes()
while not runner.is_finished():
runner.step()
assert t2.status == Trial.TERMINATED, runner.debug_string()
# Test recovery of trial that won't be checkpointed
kwargs = {
"stopping_criterion": {
"training_iteration": 3
},
"remote_checkpoint_dir": upload_dir,
}
t3 = Trial("__fake", **kwargs)
runner.add_trial(t3)
runner.step() # Start trial
runner.step() # Process result 1
cluster.add_node(num_cpus=1)
cluster.remove_node(node3)
cluster.wait_for_nodes()
while not runner.is_finished():
runner.step()
assert t3.status == Trial.ERROR, runner.debug_string()
with pytest.raises(TuneError):
runner.step()
def create_default_callbacks(callbacks: Optional[List[Callback]],
sync_config: SyncConfig,
loggers: Optional[List[Logger]]):
callbacks = callbacks or []
has_syncer_callback = False
has_csv_logger = False
has_json_logger = False
has_tbx_logger = False
# Track syncer obj/index to move callback after loggers
last_logger_index = None
syncer_index = None
# Create LegacyLoggerCallback for passed Logger classes
if loggers:
# Todo(krfricke): Deprecate `loggers` argument, print warning here.
# Add warning as soon as we ported all loggers to LoggerCallback
# classes.
add_loggers = []
for trial_logger in loggers:
if isinstance(trial_logger, LoggerCallback):
callbacks.append(trial_logger)
elif isinstance(trial_logger, type) and issubclass(
trial_logger, Logger):
add_loggers.append(trial_logger)
else:
raise ValueError(
f"Invalid value passed to `loggers` argument of "
f"`tune.run()`: {trial_logger}")
if add_loggers:
callbacks.append(LegacyLoggerCallback(add_loggers))
# Check if we have a CSV, JSON and TensorboardX logger
for i, callback in enumerate(callbacks):
if isinstance(callback, LegacyLoggerCallback):
last_logger_index = i
if CSVLogger in callback.logger_classes:
has_csv_logger = True
if JsonLogger in callback.logger_classes:
has_json_logger = True
if TBXLogger in callback.logger_classes:
has_tbx_logger = True
elif isinstance(callback, CSVLoggerCallback):
has_csv_logger = True
last_logger_index = i
elif isinstance(callback, JsonLoggerCallback):
has_json_logger = True
last_logger_index = i
elif isinstance(callback, TBXLoggerCallback):
has_tbx_logger = True
last_logger_index = i
elif isinstance(callback, SyncerCallback):
syncer_index = i
has_syncer_callback = True
# If CSV, JSON or TensorboardX loggers are missing, add
if os.environ.get("TUNE_DISABLE_AUTO_CALLBACK_LOGGERS", "0") != "1":
if not has_csv_logger:
callbacks.append(CSVLoggerCallback())
last_logger_index = len(callbacks) - 1
if not has_json_logger:
callbacks.append(JsonLoggerCallback())
last_logger_index = len(callbacks) - 1
if not has_tbx_logger:
callbacks.append(TBXLoggerCallback())
last_logger_index = len(callbacks) - 1
# If no SyncerCallback was found, add
if not has_syncer_callback and os.environ.get(
"TUNE_DISABLE_AUTO_CALLBACK_SYNCER", "0") != "1":
# Detect Docker and Kubernetes environments
_sync_to_driver = detect_sync_to_driver(sync_config.sync_to_driver)
syncer_callback = SyncerCallback(sync_function=_sync_to_driver)
callbacks.append(syncer_callback)
syncer_index = len(callbacks) - 1
if syncer_index is not None and last_logger_index is not None and \
syncer_index < last_logger_index:
if (not has_csv_logger or not has_json_logger or not has_tbx_logger) \
and not loggers:
# Only raise the warning if the loggers were passed by the user.
# (I.e. don't warn if this was automatic behavior and they only
# passed a customer SyncerCallback).
raise ValueError(
"The `SyncerCallback` you passed to `tune.run()` came before "
"at least one `LoggerCallback`. Syncing should be done "
"after writing logs. Please re-order the callbacks so that "
"the `SyncerCallback` comes after any `LoggerCallback`.")
else:
# If these loggers were automatically created. just re-order
# the callbacks
syncer_obj = callbacks[syncer_index]
callbacks.pop(syncer_index)
callbacks.insert(last_logger_index, syncer_obj)
return callbacks
def create_default_callbacks(callbacks: Optional[List[Callback]],
sync_config: SyncConfig,
loggers: Optional[List[Logger]]):
callbacks = callbacks or []
has_syncer_callback = False
has_csv_logger = False
has_json_logger = False
# Track syncer obj/index to move callback after loggers
last_logger_index = None
syncer_index = None
if not loggers:
# If no logger callback and no `loggers` have been provided,
# add DEFAULT_LOGGERS.
if not any(
isinstance(callback, ExperimentLogger)
for callback in callbacks):
loggers = DEFAULT_LOGGERS
# Create LegacyExperimentLogger for passed Logger classes
if loggers:
# Todo(krfricke): Deprecate `loggers` argument, print warning here.
add_loggers = []
for trial_logger in loggers:
if isinstance(trial_logger, ExperimentLogger):
callbacks.append(trial_logger)
elif isinstance(trial_logger, type) and issubclass(
trial_logger, Logger):
add_loggers.append(trial_logger)
else:
raise ValueError(
f"Invalid value passed to `loggers` argument of "
f"`tune.run()`: {trial_logger}")
if add_loggers:
callbacks.append(LegacyExperimentLogger(add_loggers))
# Check if we have a CSV and JSON logger
for i, callback in enumerate(callbacks):
if isinstance(callback, LegacyExperimentLogger):
last_logger_index = i
if CSVLogger in callback.logger_classes:
has_csv_logger = True
if JsonLogger in callback.logger_classes:
has_json_logger = True
# Todo(krfricke): add checks for new ExperimentLogger classes
elif isinstance(callback, SyncerCallback):
syncer_index = i
has_syncer_callback = True
# If CSV or JSON logger is missing, add
if os.environ.get("TUNE_DISABLE_AUTO_CALLBACK_LOGGERS", "0") != "1":
# Todo(krfricke): Switch to new ExperimentLogger classes
add_loggers = []
if not has_csv_logger:
add_loggers.append(CSVLogger)
if not has_json_logger:
add_loggers.append(JsonLogger)
if add_loggers:
callbacks.append(LegacyExperimentLogger(add_loggers))
last_logger_index = len(callbacks) - 1
# If no SyncerCallback was found, add
if not has_syncer_callback and os.environ.get(
"TUNE_DISABLE_AUTO_CALLBACK_SYNCER", "0") != "1":
syncer_callback = SyncerCallback(
sync_function=sync_config.sync_to_driver)
callbacks.append(syncer_callback)
syncer_index = len(callbacks) - 1
# Todo(krfricke): Maybe check if syncer comes after all loggers
if syncer_index is not None and last_logger_index is not None and \
syncer_index < last_logger_index:
if (not has_csv_logger or not has_json_logger) and not loggers:
# Only raise the warning if the loggers were passed by the user.
# (I.e. don't warn if this was automatic behavior and they only
# passed a customer SyncerCallback).
raise ValueError(
"The `SyncerCallback` you passed to `tune.run()` came before "
"at least one `ExperimentLogger`. Syncing should be done "
"after writing logs. Please re-order the callbacks so that "
"the `SyncerCallback` comes after any `ExperimentLogger`.")
else:
# If these loggers were automatically created. just re-order
# the callbacks
syncer_obj = callbacks[syncer_index]
callbacks.pop(syncer_index)
callbacks.insert(last_logger_index, syncer_obj)
return callbacks
def create_default_callbacks(
callbacks: Optional[List[Callback]],
sync_config: SyncConfig,
metric: Optional[str] = None,
):
"""Create default callbacks for `tune.run()`.
This function takes a list of existing callbacks and adds default
callbacks to it.
Specifically, three kinds of callbacks will be added:
1. Loggers. Ray Tune's experiment analysis relies on CSV and JSON logging.
2. Syncer. Ray Tune synchronizes logs and checkpoint between workers and
the head node.
2. Trial progress reporter. For reporting intermediate progress, like trial
results, Ray Tune uses a callback.
These callbacks will only be added if they don't already exist, i.e. if
they haven't been passed (and configured) by the user. A notable case
is when a Logger is passed, which is not a CSV or JSON logger - then
a CSV and JSON logger will still be created.
Lastly, this function will ensure that the Syncer callback comes after all
Logger callbacks, to ensure that the most up-to-date logs and checkpoints
are synced across nodes.
"""
callbacks = callbacks or []
has_syncer_callback = False
has_csv_logger = False
has_json_logger = False
has_tbx_logger = False
has_trial_progress_callback = any(
isinstance(c, TrialProgressCallback) for c in callbacks)
if not has_trial_progress_callback:
trial_progress_callback = TrialProgressCallback(metric=metric)
callbacks.append(trial_progress_callback)
# Track syncer obj/index to move callback after loggers
last_logger_index = None
syncer_index = None
# Check if we have a CSV, JSON and TensorboardX logger
for i, callback in enumerate(callbacks):
if isinstance(callback, LegacyLoggerCallback):
if CSVLogger in callback.logger_classes:
has_csv_logger = True
if JsonLogger in callback.logger_classes:
has_json_logger = True
if TBXLogger in callback.logger_classes:
has_tbx_logger = True
elif isinstance(callback, CSVLoggerCallback):
has_csv_logger = True
elif isinstance(callback, JsonLoggerCallback):
has_json_logger = True
elif isinstance(callback, TBXLoggerCallback):
has_tbx_logger = True
elif isinstance(callback, SyncerCallback):
syncer_index = i
has_syncer_callback = True
if isinstance(callback, LoggerCallback):
last_logger_index = i
# If CSV, JSON or TensorboardX loggers are missing, add
if os.environ.get("TUNE_DISABLE_AUTO_CALLBACK_LOGGERS", "0") != "1":
if not has_csv_logger:
callbacks.append(CSVLoggerCallback())
last_logger_index = len(callbacks) - 1
if not has_json_logger:
callbacks.append(JsonLoggerCallback())
last_logger_index = len(callbacks) - 1
if not has_tbx_logger:
try:
callbacks.append(TBXLoggerCallback())
last_logger_index = len(callbacks) - 1
except ImportError:
logger.warning(
"The TensorboardX logger cannot be instantiated because "
"either TensorboardX or one of it's dependencies is not "
"installed. Please make sure you have the latest version "
"of TensorboardX installed: `pip install -U tensorboardx`")
# If no SyncerCallback was found, add
if (not has_syncer_callback and
os.environ.get("TUNE_DISABLE_AUTO_CALLBACK_SYNCER", "0") != "1"):
syncer_callback = SyncerCallback(enabled=bool(sync_config.syncer),
sync_period=sync_config.sync_period)
callbacks.append(syncer_callback)
syncer_index = len(callbacks) - 1
if (syncer_index is not None and last_logger_index is not None
and syncer_index < last_logger_index):
# Re-order callbacks
syncer_obj = callbacks[syncer_index]
callbacks.pop(syncer_index)
callbacks.insert(last_logger_index, syncer_obj)
return callbacks