def run_noise_test(config): """Run noise test on the best scoring model found during training. Make sure to train the models before calling this function. :param config: The configuration of the pre-trained model. :return: dict with noise test results over all experiments """ # Load experiment data name = config["name"] experiment_path = os.path.join(config["path"], name) experiment_state = load_ray_tune_experiment( experiment_path=experiment_path, load_results=True) # Go through all checkpoints in the experiment all_checkpoints = experiment_state["checkpoints"] for checkpoint in all_checkpoints: results = checkpoint["results"] if results is None: continue # For each checkpoint select the epoch with the best accuracy as the best epoch best_result = max(results, key=lambda x: x["mean_accuracy"]) best_epoch = best_result["training_iteration"] best_config = best_result["config"] print("best_epoch: ", best_epoch) # Update path best_config["path"] = config["path"] best_config["data_dir"] = config["data_dir"] # Load pre-trained model from checkpoint and run noise test on it logdir = os.path.join(experiment_path, os.path.basename(checkpoint["logdir"])) checkpoint_path = os.path.join(logdir, "checkpoint_{}".format(best_epoch)) experiment = SparseSpeechExperiment(best_config) experiment.restore(checkpoint_path) # Save noise results in checkpoint log dir noise_test = os.path.join(logdir, "noise.json") with open(noise_test, "w") as f: res = experiment.run_noise_tests() json.dump(res, f) # Compute total noise score total_correct = 0 for k, v in res.items(): print(k, v, v["total_correct"]) total_correct += v["total_correct"] print("Total across all noise values", total_correct) # Upload results to S3 sync_function = config.get("sync_function", None) if sync_function is not None: upload_dir = config["upload_dir"] final_cmd = sync_function.format(local_dir=experiment_path, remote_dir=upload_dir) subprocess.Popen(final_cmd, shell=True)
def run_noise_test(config): """ Run noise test on the best scoring model found during training. Make sure to train the models before calling this function :param config: The configuration of the pre-trained model. :return: dict with noise test results over all experiments """ # Load experiment data name = config["name"] experiment_path = os.path.join(config["path"], name) experiment_state = load_ray_tune_experiment( experiment_path=experiment_path, load_results=True) # Go through all checkpoints in the experiment all_checkpoints = experiment_state["checkpoints"] for checkpoint in all_checkpoints: results = checkpoint["results"] if results is None: continue # For each checkpoint select the epoch with the best accuracy as the best epoch best_result = max(results, key=lambda x: x["mean_accuracy"]) best_epoch = best_result["training_iteration"] best_config = best_result["config"] # Update path best_config["path"] = config["path"] best_config["data_dir"] = config["data_dir"] # Load pre-trained model from checkpoint and run noise test on it logdir = os.path.join(experiment_path, os.path.basename(checkpoint["logdir"])) checkpoint_path = os.path.join(logdir, "checkpoint_{}".format(best_epoch)) experiment = SparseSpeechExperiment(best_config) experiment.restore(checkpoint_path) # Save noise results in checkpoint log dir noise_test = os.path.join(logdir, "noise.json") with open(noise_test, "w") as f: json.dump(experiment.runNoiseTests(), f) # Upload results to S3 sync_function = config.get("sync_function", None) if sync_function is not None: upload_dir = config["upload_dir"] final_cmd = sync_function.format(local_dir=experiment_path, remote_dir=upload_dir) subprocess.Popen(final_cmd, shell=True)
class SpeechExperimentTune(tune.Trainable): """Ray tune trainable wrapping MNISTSparseExperiment.""" def _setup(self, config): self.experiment = SparseSpeechExperiment(config) def _train(self): self.experiment.train(self._iteration) return self.experiment.test() def _save(self, checkpoint_dir): return self.experiment.save(checkpoint_dir) def _restore(self, checkpoint_dir): self.experiment.restore(checkpoint_dir)
class SpeechExperimentTune(tune.Trainable): """ Ray tune trainable wrapping MNISTSparseExperiment """ def _setup(self, config): self.experiment = SparseSpeechExperiment(config) def _train(self): self.experiment.train(self._iteration) return self.experiment.test() def _save(self, checkpoint_dir): return self.experiment.save(checkpoint_dir) def _restore(self, checkpoint_dir): self.experiment.restore(checkpoint_dir)
def _setup(self, config): self.experiment = SparseSpeechExperiment(config)
def _setup(self, config): self.experiment = SparseSpeechExperiment(config)