def testBestLogdir(self):
analysis = ExperimentAnalysis(self.test_dir)
logdir = analysis.get_best_logdir(self.metric, mode="max")
self.assertTrue(logdir.startswith(self.test_dir))
logdir2 = analysis.get_best_logdir(self.metric, mode="min")
self.assertTrue(logdir2.startswith(self.test_dir))
self.assertNotEqual(logdir, logdir2)
def load_best_model(exp: tune.ExperimentAnalysis,
metric="val_loss",
mode="min"):
"""Loads best checkpoint overall"""
tr = exp.get_best_trial(metric=metric, mode=mode, scope="all")
chkpoint = exp.get_best_checkpoint(tr, metric=metric, mode=mode)
model = CLIPFineTunedModel.load_from_checkpoint(chkpoint + "/checkpoint")
return model
def register_analysis(self, analysis: ExperimentAnalysis):
"""Integrate the given analysis into the gaussian process.
Args:
analysis (ExperimentAnalysis): Optionally, the previous analysis
to integrate.
"""
for (_, report), params in zip(
analysis.dataframe(metric=self._metric,
mode=self._mode).iterrows(),
analysis.get_all_configs().values()):
# We add the obtained results to the
# gaussian process optimizer
self._register_result(params, report)
def testInitLegacy(self):
"""Should still work if checkpoints are not json strings"""
experiment_checkpoint_path = os.path.join(self.test_dir,
"experiment_state.json")
checkpoint_data = {
"checkpoints": [{
"trial_id":
"abcd1234",
"status":
Trial.TERMINATED,
"trainable_name":
"MockTrainable",
"local_dir":
self.test_dir,
"relative_logdir":
"MockTrainable_0_id=3_2020-07-12",
}]
}
with open(experiment_checkpoint_path, "w") as f:
f.write(json.dumps(checkpoint_data))
experiment_analysis = ExperimentAnalysis(experiment_checkpoint_path)
self.assertEqual(len(experiment_analysis._checkpoints_and_paths), 1)
self.assertTrue(experiment_analysis.trials)
def testInit(self):
experiment_checkpoint_path = os.path.join(
self.test_dir, "experiment_state.json"
)
checkpoint_data = {
"checkpoints": [
json.dumps(
{
"trial_id": "abcd1234",
"status": Trial.TERMINATED,
"trainable_name": "MockTrainable",
"local_dir": self.test_dir,
"relative_logdir": "MockTrainable_0_id=3_2020-07-12",
},
cls=TuneFunctionEncoder,
)
]
}
with open(experiment_checkpoint_path, "w") as f:
f.write(json.dumps(checkpoint_data))
experiment_analysis = ExperimentAnalysis(experiment_checkpoint_path)
self.assertEqual(len(experiment_analysis._checkpoints_and_paths), 1)
self.assertTrue(experiment_analysis.trials)
def raytune_analysis(exp_dir, save, skip, mode, metric):
from ray.tune import ExperimentAnalysis
experiment_analysis = ExperimentAnalysis(exp_dir,
default_metric=metric,
default_mode=mode)
plot_ray_analysis(experiment_analysis, save=save, skip=skip)
analyze_ray_experiment(exp_dir, default_metric=metric, default_mode=mode)
async def collect(self):
"""
Collects and cleans data on the running Tune experiment from the
Tune logs so that users can see this information in the front-end
client
"""
self._trial_records = {}
self._errors = {}
if not self._logdir or not ExperimentAnalysis:
return
# search through all the sub_directories in log directory
analysis = ExperimentAnalysis(str(self._logdir))
df = analysis.dataframe(metric=None, mode=None)
if len(df) == 0 or "trial_id" not in df.columns:
return
self._trials_available = True
# make sure that data will convert to JSON without error
df["trial_id_key"] = df["trial_id"].astype(str)
df = df.fillna(0)
trial_ids = df["trial_id"]
for i, value in df["trial_id"].iteritems():
if type(value) != str and type(value) != int:
trial_ids[i] = int(value)
df["trial_id"] = trial_ids
# convert df to python dict
df = df.set_index("trial_id_key")
trial_data = df.to_dict(orient="index")
# clean data and update class attribute
if len(trial_data) > 0:
trial_data = self.clean_trials(trial_data)
self._trial_records.update(trial_data)
self.collect_errors(df)
def testInit(self):
experiment_checkpoint_path = os.path.join(self.test_dir,
"experiment_state.json")
checkpoint_data = {
"checkpoints": [{
"trainable_name": "MockTrainable",
"logdir": "/mock/test/MockTrainable_0_id=3_2020-07-12"
}]
}
with open(experiment_checkpoint_path, "w") as f:
f.write(json.dumps(checkpoint_data))
experiment_analysis = ExperimentAnalysis(experiment_checkpoint_path)
self.assertEqual(len(experiment_analysis._checkpoints), 1)
self.assertTrue(experiment_analysis.trials is None)
def testInitLegacy(self):
"""Should still work if checkpoints are not json strings"""
experiment_checkpoint_path = os.path.join(self.test_dir,
"experiment_state.json")
checkpoint_data = {
"checkpoints": [{
"trainable_name":
"MockTrainable",
"logdir":
"/mock/test/MockTrainable_0_id=3_2020-07-12"
}]
}
with open(experiment_checkpoint_path, "w") as f:
f.write(json.dumps(checkpoint_data))
experiment_analysis = ExperimentAnalysis(experiment_checkpoint_path)
self.assertEqual(len(experiment_analysis._checkpoints), 1)
self.assertFalse(experiment_analysis.trials)
def testFromPath(self):
self.run_test_exp()
analysis = ExperimentAnalysis(self.test_path)
self.assertTrue(analysis.get_best_trial(metric=self.metric, mode="max"))
ray.shutdown()
ray.tune.registry._global_registry = ray.tune.registry._Registry(
prefix="global"
)
analysis = ExperimentAnalysis(self.test_path)
# This will be None if validate_trainable during loading fails
self.assertTrue(analysis.get_best_trial(metric=self.metric, mode="max"))
def testInitException(self):
experiment_checkpoint_path = os.path.join(self.test_dir, "mock.json")
with pytest.raises(ValueError):
ExperimentAnalysis(experiment_checkpoint_path)
def testBestConfigIsLogdir(self):
analysis = ExperimentAnalysis(self.test_dir)
for metric, mode in [(self.metric, "min"), (self.metric, "max")]:
logdir = analysis.get_best_logdir(metric, mode=mode)
best_config = analysis.get_best_config(metric, mode=mode)
self.assertEqual(analysis.get_all_configs()[logdir], best_config)
def testDataframe(self):
analysis = ExperimentAnalysis(self.test_dir)
df = analysis.dataframe(self.metric, mode="max")
self.assertTrue(isinstance(df, pd.DataFrame))
self.assertEqual(df.shape[0], self.num_samples * 2)
def get_best_trial(analysis: tune.ExperimentAnalysis, objective: Objective, scope: str):
return analysis.get_best_trial(
full_metric_name(objective), mode=objective.mode, scope=scope
)
