def plot(config, experiments):
print("config =", config.name)
print("experiments =", experiments)
# Use configuration file location as the project location.
project_dir = os.path.dirname(config.name)
project_dir = os.path.abspath(project_dir)
print("project_dir =", project_dir)
# Load and parse experiment configurations
noise_experiments = defaultdict(list)
configs = parse_config(config, experiments, globals(), locals())
for exp in configs:
config = configs[exp]
# Load experiment state and get all the tags
experiment_path = os.path.join(project_dir, config["path"], exp)
experiment_state = load_ray_tune_experiment(experiment_path)
all_experiments = experiment_state["checkpoints"]
for experiment in all_experiments:
noise_experiments[exp].append(experiment["experiment_tag"])
# Plot noise experiments
for exp in noise_experiments:
fig, ax = plt.subplots()
for tag in noise_experiments[exp]:
# Load experiment results
experiment_path = os.path.join(project_dir, "results", "noise",
exp, tag)
if not os.path.exists(experiment_path):
continue
experiment_state = load_ray_tune_experiment(experiment_path,
load_results=True)
all_experiments = experiment_state["checkpoints"]
data = {}
for experiment in all_experiments:
acc = experiment["results"][0]["mean_accuracy"]
noise = experiment["config"]["noise"]
data.setdefault(noise, acc)
data = OrderedDict(sorted(data.items(), key=lambda i: i[0]))
ax.plot(list(data.keys()), list(data.values()), label=tag)
fig.suptitle("Accuracy vs noise")
ax.set_xlabel("Noise")
ax.set_ylabel("Accuracy (percent)")
plt.legend()
plt.grid(axis="y")
plot_path = os.path.join(project_dir, "results", "noise",
"{}_noise.pdf".format(exp))
plt.savefig(plot_path)
plt.close()
def plot(config, experiments):
print("config =", config.name)
print("experiments =", experiments)
# Use configuration file location as the project location.
projectDir = os.path.dirname(config.name)
projectDir = os.path.abspath(projectDir)
print("projectDir =", projectDir)
# Load and parse experiment configurations
noise_experiments = defaultdict(list)
configs = parse_config(config, experiments, globals(), locals())
for exp in configs:
config = configs[exp]
# Load experiment state and get all the tags
experiment_path = os.path.join(projectDir, config["path"], exp)
experiment_state = load_ray_tune_experiment(experiment_path)
all_experiments = experiment_state["checkpoints"]
for experiment in all_experiments:
noise_experiments[exp].append(experiment["experiment_tag"])
# Plot noise experiments
for exp in noise_experiments:
fig, ax = plt.subplots()
for tag in noise_experiments[exp]:
# Load experiment results
experiment_path = os.path.join(projectDir, "results", "noise", exp, tag)
if not os.path.exists(experiment_path):
continue
experiment_state = load_ray_tune_experiment(experiment_path, load_results=True)
all_experiments = experiment_state["checkpoints"]
data = {}
for experiment in all_experiments:
acc = experiment["results"][0]["mean_accuracy"]
noise = experiment["config"]["noise"]
data.setdefault(noise, acc)
data = OrderedDict(sorted(data.items(), key=lambda i: i[0]))
ax.plot(list(data.keys()), list(data.values()), label=tag)
fig.suptitle("Accuracy vs noise")
ax.set_xlabel("Noise")
ax.set_ylabel("Accuracy (percent)")
plt.legend()
plt.grid(axis='y')
plot_path = os.path.join(projectDir, "results", "noise", "{}_noise.pdf".format(exp))
plt.savefig(plot_path)
plt.close()
def main(config):
# Use configuration file location as the project location.
project_dir = os.path.dirname(config.name)
project_dir = os.path.abspath(project_dir)
# Plot noisy images
data_dir = os.path.join(project_dir, "data")
plotPath = os.path.join(project_dir, "mnist_images_with_noise.pdf")
plotImagesWithNoise(datadir=data_dir,
noise_values=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5],
plotPath=plotPath)
# Load and parse experiment configurations
configs = parse_config(config_file=config,
experiments=list(EXPERIMENTS.keys()),
globals=globals())
results = {}
for exp in configs:
config = configs[exp]
# Load experiment data
experiment_path = os.path.join(project_dir, config["path"], exp)
experiment_state = load_ray_tune_experiment(
experiment_path=experiment_path, load_results=True)
# Load noise score from the first checkpoint
checkpoint = experiment_state["checkpoints"][0]
logdir = os.path.join(experiment_path, os.path.basename(checkpoint["logdir"]))
filename = os.path.join(logdir, "noise.json")
with open(filename, "r") as f:
results[exp] = json.load(f)
plotPath = os.path.join(project_dir, "accuracy_vs_noise.pdf")
plotNoiseCurve(configs=configs, results=results, plotPath=plotPath)
def run_noise_test(config):
"""Run noise test on the best scoring model found during training. Make
sure to train the models before calling this function.
:param config: The configuration of the pre-trained model.
:return: dict with noise test results over all experiments
"""
# Load experiment data
name = config["name"]
experiment_path = os.path.join(config["path"], name)
experiment_state = load_ray_tune_experiment(
experiment_path=experiment_path, load_results=True)
# Go through all checkpoints in the experiment
all_checkpoints = experiment_state["checkpoints"]
for checkpoint in all_checkpoints:
results = checkpoint["results"]
if results is None:
continue
# For each checkpoint select the epoch with the best accuracy as the best epoch
best_result = max(results, key=lambda x: x["mean_accuracy"])
best_epoch = best_result["training_iteration"]
best_config = best_result["config"]
print("best_epoch: ", best_epoch)
# Update path
best_config["path"] = config["path"]
best_config["data_dir"] = config["data_dir"]
# Load pre-trained model from checkpoint and run noise test on it
logdir = os.path.join(experiment_path,
os.path.basename(checkpoint["logdir"]))
checkpoint_path = os.path.join(logdir,
"checkpoint_{}".format(best_epoch))
experiment = SparseSpeechExperiment(best_config)
experiment.restore(checkpoint_path)
# Save noise results in checkpoint log dir
noise_test = os.path.join(logdir, "noise.json")
with open(noise_test, "w") as f:
res = experiment.run_noise_tests()
json.dump(res, f)
# Compute total noise score
total_correct = 0
for k, v in res.items():
print(k, v, v["total_correct"])
total_correct += v["total_correct"]
print("Total across all noise values", total_correct)
# Upload results to S3
sync_function = config.get("sync_function", None)
if sync_function is not None:
upload_dir = config["upload_dir"]
final_cmd = sync_function.format(local_dir=experiment_path,
remote_dir=upload_dir)
subprocess.Popen(final_cmd, shell=True)
def main(config, experiments, tablefmt):
# Use configuration file location as the project location.
project_dir = os.path.dirname(config.name)
project_dir = os.path.abspath(project_dir)
print("project_dir =", project_dir)
testScoresTable = [["Network", "Test Score", "Noise Score"]]
# Load and parse experiment configurations
configs = parse_config(config, experiments, globals=globals())
# Select tags ignoring seed
key_func = lambda x: re.split("[,_]", re.sub(",|\\d+_|seed=\\d+", "",
x["experiment_tag"]))
for exp in configs:
config = configs[exp]
# Load experiment data
experiment_path = os.path.join(project_dir, config["path"], exp)
experiment_state = load_ray_tune_experiment(
experiment_path=experiment_path, load_results=True)
# Go through all checkpoints in the experiment
all_checkpoints = experiment_state["checkpoints"]
# Group checkpoints by tags
checkpoint_groups = {k[0]: list(v) for k, v in groupby(
sorted(all_checkpoints, key=key_func), key=key_func)}
for tag in checkpoint_groups:
checkpoints = checkpoint_groups[tag]
numExps = len(checkpoints)
testScores = np.zeros(numExps)
noiseScores = np.zeros(numExps)
for i, checkpoint in enumerate(checkpoints):
results = checkpoint["results"]
if results is None:
continue
# For each checkpoint select the epoch with the best accuracy as the best epoch
best_result = max(results, key=lambda x: x["mean_accuracy"])
testScores[i] = best_result["mean_accuracy"] * 100.0
# Load noise score
logdir = os.path.join(experiment_path, os.path.basename(checkpoint["logdir"]))
filename = os.path.join(logdir, "noise.json")
with open(filename, "r") as f:
noise = json.load(f)
noiseScores[i] = sum([x["total_correct"] for x in list(noise.values())])
test_score = u"{0:.2f} ± {1:.2f}".format(testScores.mean(), testScores.std())
noise_score = u"{0:,.0f} ± {1:.2f}".format(noiseScores.mean(), noiseScores.std())
testScoresTable.append(["{} {}".format(exp, tag), test_score, noise_score])
print()
print(tabulate(testScoresTable, headers="firstrow", tablefmt=tablefmt))
def run_noise_test(config):
"""
Run noise test on the best scoring model found during training. Make sure to
train the models before calling this function
:param config: The configuration of the pre-trained model.
:return: dict with noise test results over all experiments
"""
# Load experiment data
name = config["name"]
experiment_path = os.path.join(config["path"], name)
experiment_state = load_ray_tune_experiment(
experiment_path=experiment_path, load_results=True)
# Go through all checkpoints in the experiment
all_checkpoints = experiment_state["checkpoints"]
for checkpoint in all_checkpoints:
results = checkpoint["results"]
if results is None:
continue
# For each checkpoint select the epoch with the best accuracy as the best epoch
best_result = max(results, key=lambda x: x["mean_accuracy"])
best_epoch = best_result["training_iteration"]
best_config = best_result["config"]
# Update path
best_config["path"] = config["path"]
best_config["data_dir"] = config["data_dir"]
# Load pre-trained model from checkpoint and run noise test on it
logdir = os.path.join(experiment_path, os.path.basename(checkpoint["logdir"]))
checkpoint_path = os.path.join(logdir, "checkpoint_{}".format(best_epoch))
experiment = SparseSpeechExperiment(best_config)
experiment.restore(checkpoint_path)
# Save noise results in checkpoint log dir
noise_test = os.path.join(logdir, "noise.json")
with open(noise_test, "w") as f:
json.dump(experiment.runNoiseTests(), f)
# Upload results to S3
sync_function = config.get("sync_function", None)
if sync_function is not None:
upload_dir = config["upload_dir"]
final_cmd = sync_function.format(local_dir=experiment_path,
remote_dir=upload_dir)
subprocess.Popen(final_cmd, shell=True)
def main(config):
# Use configuration file location as the project location.
project_dir = Path(dirname(config.name)).expanduser().resolve()
data_dir = Path(project_dir) / "data"
# Load and parse experiment configurations
configs = parse_config(
config_file=config,
experiments=list(EXPERIMENTS.keys()),
globals_param=globals(),
)
results = {}
for exp in configs:
config = configs[exp]
# Load experiment data
data_dir = Path(config["data_dir"]).expanduser().resolve()
path = Path(config["path"]).expanduser().resolve()
experiment_path = path / exp
experiment_state = load_ray_tune_experiment(
experiment_path=experiment_path, load_results=True)
# Load noise score and compute the mean_accuracy over all checkpoints
exp_df = pd.DataFrame()
for checkpoint in experiment_state["checkpoints"]:
logdir = experiment_path / basename(checkpoint["logdir"])
filename = logdir / "noise.json"
with open(filename, "r") as f:
df = pd.DataFrame(json.load(f)).transpose()
exp_df = exp_df.append(df["mean_accuracy"], ignore_index=True)
results[exp] = exp_df.mean()
plot_path = project_dir / "accuracy_vs_noise.pdf"
plot_noise_curve(configs=configs, results=results, plot_path=plot_path)
# Plot noisy images
plot_path = project_dir / "mnist_images_with_noise.pdf"
plot_images_with_noise(
datadir=data_dir,
noise_values=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5],
plot_path=plot_path,
)
def run_noise_test(config):
"""Run noise test on the best scoring model found during training. Make
sure to train the models before calling this function.
:param config: The configuration of the pre-trained model.
:return: dict with noise test results over all experiments
"""
# Load experiment data
name = config["name"]
experiment_path = os.path.join(config["path"], name)
experiment_state = load_ray_tune_experiment(
experiment_path=experiment_path, load_results=True
)
# Go through all checkpoints in the experiment
all_checkpoints = experiment_state["checkpoints"]
for checkpoint in all_checkpoints:
results = checkpoint["results"]
if results is None:
continue
# For each checkpoint select the epoch with the best accuracy as the best epoch
best_result = max(results, key=lambda x: x["mean_accuracy"])
best_epoch = best_result["training_iteration"]
# Load pre-trained model from checkpoint and run noise test on it
logdir = os.path.join(experiment_path, os.path.basename(checkpoint["logdir"]))
checkpoint_path = os.path.join(logdir, "checkpoint_{}".format(best_epoch))
# Get the actual config from the saved version (required for sample
# or grid search experiments). Replace paths to be the locally correct ones
filename = os.path.join(logdir, "params.json")
with open(filename, "r") as f:
saved_params = json.load(f)
saved_params["data_dir"] = config["data_dir"]
saved_params["path"] = config["path"]
experiment = MNISTSparseExperiment(saved_params)
experiment.restore(checkpoint_path)
# Save noise results in checkpoint log dir
noise_test = os.path.join(logdir, "noise.json")
with open(noise_test, "w") as f:
json.dump(experiment.run_noise_tests(), f)
def main(config):
# Use configuration file location as the project location.
project_dir = os.path.dirname(config.name)
project_dir = os.path.abspath(project_dir)
# Plot noisy images
data_dir = os.path.join(project_dir, "data")
plot_path = os.path.join(project_dir, "mnist_images_with_noise.pdf")
plot_images_with_noise(
datadir=data_dir,
noise_values=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5],
plot_path=plot_path,
)
# Load and parse experiment configurations
configs = parse_config(
config_file=config,
experiments=list(EXPERIMENTS.keys()),
globals_param=globals(),
)
results = {}
for exp in configs:
config = configs[exp]
# Load experiment data
experiment_path = os.path.join(project_dir, config["path"], exp)
experiment_state = load_ray_tune_experiment(
experiment_path=experiment_path, load_results=True)
# Load noise score from the first checkpoint
checkpoint = experiment_state["checkpoints"][0]
logdir = os.path.join(experiment_path,
os.path.basename(checkpoint["logdir"]))
filename = os.path.join(logdir, "noise.json")
with open(filename, "r") as f:
results[exp] = json.load(f)
plot_path = os.path.join(project_dir, "accuracy_vs_noise.pdf")
plot_noise_curve(configs=configs, results=results, plot_path=plot_path)
def main(config, experiments, tablefmt):
# Use configuration file location as the project location.
project_dir = os.path.dirname(config.name)
project_dir = os.path.abspath(project_dir)
print("project_dir =", project_dir)
test_scores_table = [["Network", "Test Score", "Noise Score"]]
# Load and parse experiment configurations
configs = parse_config(config, experiments, globals_param=globals())
# Select tags ignoring seed
def key_func(x):
re.split("[,_]", re.sub(",|\\d+_|seed=\\d+", "", x["experiment_tag"]))
for exp in configs:
config = configs[exp]
# Load experiment data
experiment_path = os.path.join(project_dir, config["path"], exp)
experiment_state = load_ray_tune_experiment(
experiment_path=experiment_path, load_results=True)
# Go through all checkpoints in the experiment
all_checkpoints = experiment_state["checkpoints"]
# Group checkpoints by tags
checkpoint_groups = {
k[0]: list(v)
for k, v in groupby(sorted(all_checkpoints, key=key_func),
key=key_func)
}
for tag in checkpoint_groups:
checkpoints = checkpoint_groups[tag]
num_exps = len(checkpoints)
test_scores = np.zeros(num_exps)
noise_scores = np.zeros(num_exps)
for i, checkpoint in enumerate(checkpoints):
results = checkpoint["results"]
if results is None:
continue
# For each checkpoint select the epoch with the best accuracy as
# the best epoch
best_result = max(results, key=lambda x: x["mean_accuracy"])
test_scores[i] = best_result["mean_accuracy"] * 100.0
# Load noise score
logdir = os.path.join(experiment_path,
os.path.basename(checkpoint["logdir"]))
filename = os.path.join(logdir, "noise.json")
with open(filename, "r") as f:
noise = json.load(f)
noise_scores[i] = sum(x["total_correct"]
for x in list(noise.values()))
test_score = "{0:.2f} ± {1:.2f}".format(test_scores.mean(),
test_scores.std())
noise_score = "{0:,.0f} ± {1:.2f}".format(noise_scores.mean(),
noise_scores.std())
test_scores_table.append(
["{} {}".format(exp, tag), test_score, noise_score])
print()
print(tabulate(test_scores_table, headers="firstrow", tablefmt=tablefmt))
def run(config, experiments, num_cpus, num_gpus, redis_address, noise_values):
print("config =", config.name)
print("experiments =", experiments)
print("num_gpus =", num_gpus)
print("num_cpus =", num_cpus)
print("redis_address =", redis_address)
print("noise_values =", noise_values)
# Use configuration file location as the project location.
project_dir = os.path.dirname(config.name)
project_dir = os.path.abspath(project_dir)
print("project_dir =", project_dir)
# Download dataset
data_dir = os.path.join(project_dir, "data")
datasets.CIFAR10(data_dir, download=True, train=True)
# Initialize ray cluster
if redis_address is not None:
ray.init(redis_address=redis_address, include_webui=True)
else:
ray.init(num_cpus=num_cpus,
num_gpus=num_gpus,
local_mode=num_cpus == 1)
# Load and parse experiment configurations
configs = parse_config(config, experiments, globals(), locals())
# Run all experiments in parallel
ray_trials = []
for exp in configs:
config = configs[exp]
# noise experiment tune configuration
noise_config = {
"iterations": 1,
"noise": {
"grid_search": list(noise_values)
}
}
# Download results from S3 when running on the cluster
if redis_address is not None and "upload_dir" in config:
upload_dir = config["upload_dir"]
download_s3_results("{}/{}".format(upload_dir, exp),
os.path.join(config["path"], exp))
# Store noise results with original results in S3
noise_config["upload_dir"] = "{}".format(upload_dir)
noise_config[
"sync_function"] = "aws s3 sync `dirname {local_dir}` {remote_dir}/`basename $(dirname {local_dir})`" # noqa E501
else:
noise_config.pop("upload_dir", None)
noise_config.pop("sync_function", None)
# Load experiment results
experiment_path = os.path.join(project_dir, config["path"], exp)
experiment_state = load_ray_tune_experiment(experiment_path,
load_results=True)
all_experiments = experiment_state["checkpoints"]
for experiment in all_experiments:
# Make logs relative to experiment path
logdir = experiment["logdir"]
logpath = os.path.join(experiment_path, os.path.basename(logdir))
# Check for experiment results
results = experiment["results"]
if results is None:
continue
# Get best scoring model checkpoint from results
best_result = max(results, key=lambda x: x["mean_accuracy"])
epoch = best_result["training_iteration"]
checkpoint_path = os.path.join(logpath,
"checkpoint_{}".format(epoch))
if os.path.exists(checkpoint_path):
# Update data path
model_config = best_result["config"]
model_config["data_dir"] = data_dir
# Run noise tests
noise_config.update({
"name":
experiment["experiment_tag"],
"path":
os.path.join(project_dir, "results", "noise", exp),
"checkpoint_path":
checkpoint_path,
"model_config":
model_config,
})
ray_trials.append(
run_experiment.remote(
noise_config,
MobileNetNoiseTune,
num_cpus=1,
num_gpus=min(1, num_gpus),
))
# Wait for all experiments to complete
ray.get(ray_trials)
ray.shutdown()
def run(config, experiments, num_cpus, num_gpus, redis_address, noise_values):
print("config =", config.name)
print("experiments =", experiments)
print("num_gpus =", num_gpus)
print("num_cpus =", num_cpus)
print("redis_address =", redis_address)
print("noise_values =", noise_values)
# Use configuration file location as the project location.
projectDir = os.path.dirname(config.name)
projectDir = os.path.abspath(projectDir)
print("projectDir =", projectDir)
# Download dataset
data_dir = os.path.join(projectDir, "data")
datasets.CIFAR10(data_dir, download=True, train=True)
# Initialize ray cluster
if redis_address is not None:
ray.init(redis_address=redis_address, include_webui=True)
else:
ray.init(num_cpus=num_cpus, num_gpus=num_gpus, local_mode=num_cpus == 1)
# Load and parse experiment configurations
configs = parse_config(config, experiments, globals(), locals())
# Run all experiments in parallel
ray_trials = []
for exp in configs:
config = configs[exp]
# noise experiment tune configuration
noise_config = {
"iterations": 1,
"noise": {"grid_search": list(noise_values)}
}
# Download results from S3 when running on the cluster
if redis_address is not None and "upload_dir" in config:
upload_dir = config["upload_dir"]
download_s3_results("{}/{}".format(upload_dir, exp),
os.path.join(config["path"], exp))
# Store noise results with original results in S3
noise_config["upload_dir"] = "{}".format(upload_dir)
noise_config["sync_function"] = "aws s3 sync `dirname {local_dir}` {remote_dir}/`basename $(dirname {local_dir})`"
else:
noise_config.pop("upload_dir", None)
noise_config.pop("sync_function", None)
# Load experiment results
experiment_path = os.path.join(projectDir, config["path"], exp)
experiment_state = load_ray_tune_experiment(experiment_path,
load_results=True)
all_experiments = experiment_state["checkpoints"]
for experiment in all_experiments:
# Make logs relative to experiment path
logdir = experiment["logdir"]
logpath = os.path.join(experiment_path, os.path.basename(logdir))
# Check for experiment results
results = experiment["results"]
if results is None:
continue
# Get best scoring model checkpoint from results
best_result = max(results, key=lambda x: x["mean_accuracy"])
epoch = best_result["training_iteration"]
checkpoint_path = os.path.join(logpath, "checkpoint_{}".format(epoch))
if os.path.exists(checkpoint_path):
# Update data path
model_config = best_result["config"]
model_config["data_dir"] = data_dir
# Run noise tests
noise_config.update({
"name": experiment["experiment_tag"],
"path": os.path.join(projectDir, "results", "noise", exp),
"checkpoint_path": checkpoint_path,
"model_config": model_config
})
ray_trials.append(
run_experiment.remote(noise_config, MobileNetNoiseTune,
num_cpus=1, num_gpus=min(1, num_gpus)))
# Wait for all experiments to complete
ray.get(ray_trials)
ray.shutdown()