def main(config):
# Use configuration file location as the project location.
project_dir = os.path.dirname(config.name)
project_dir = os.path.abspath(project_dir)
# Plot noisy images
data_dir = os.path.join(project_dir, "data")
plotPath = os.path.join(project_dir, "mnist_images_with_noise.pdf")
plotImagesWithNoise(datadir=data_dir,
noise_values=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5],
plotPath=plotPath)
# Load and parse experiment configurations
configs = parse_config(config_file=config,
experiments=list(EXPERIMENTS.keys()),
globals=globals())
results = {}
for exp in configs:
config = configs[exp]
# Load experiment data
experiment_path = os.path.join(project_dir, config["path"], exp)
experiment_state = load_ray_tune_experiment(
experiment_path=experiment_path, load_results=True)
# Load noise score from the first checkpoint
checkpoint = experiment_state["checkpoints"][0]
logdir = os.path.join(experiment_path, os.path.basename(checkpoint["logdir"]))
filename = os.path.join(logdir, "noise.json")
with open(filename, "r") as f:
results[exp] = json.load(f)
plotPath = os.path.join(project_dir, "accuracy_vs_noise.pdf")
plotNoiseCurve(configs=configs, results=results, plotPath=plotPath)
def main(config, experiments, tablefmt):
# Use configuration file location as the project location.
project_dir = os.path.dirname(config.name)
project_dir = os.path.abspath(project_dir)
print("project_dir =", project_dir)
testScoresTable = [["Network", "Test Score", "Noise Score"]]
# Load and parse experiment configurations
configs = parse_config(config, experiments, globals=globals())
# Select tags ignoring seed
key_func = lambda x: re.split("[,_]", re.sub(",|\\d+_|seed=\\d+", "",
x["experiment_tag"]))
for exp in configs:
config = configs[exp]
# Load experiment data
experiment_path = os.path.join(project_dir, config["path"], exp)
experiment_state = load_ray_tune_experiment(
experiment_path=experiment_path, load_results=True)
# Go through all checkpoints in the experiment
all_checkpoints = experiment_state["checkpoints"]
# Group checkpoints by tags
checkpoint_groups = {k[0]: list(v) for k, v in groupby(
sorted(all_checkpoints, key=key_func), key=key_func)}
for tag in checkpoint_groups:
checkpoints = checkpoint_groups[tag]
numExps = len(checkpoints)
testScores = np.zeros(numExps)
noiseScores = np.zeros(numExps)
for i, checkpoint in enumerate(checkpoints):
results = checkpoint["results"]
if results is None:
continue
# For each checkpoint select the epoch with the best accuracy as the best epoch
best_result = max(results, key=lambda x: x["mean_accuracy"])
testScores[i] = best_result["mean_accuracy"] * 100.0
# Load noise score
logdir = os.path.join(experiment_path, os.path.basename(checkpoint["logdir"]))
filename = os.path.join(logdir, "noise.json")
with open(filename, "r") as f:
noise = json.load(f)
noiseScores[i] = sum([x["total_correct"] for x in list(noise.values())])
test_score = u"{0:.2f} ± {1:.2f}".format(testScores.mean(), testScores.std())
noise_score = u"{0:,.0f} ± {1:.2f}".format(noiseScores.mean(), noiseScores.std())
testScoresTable.append(["{} {}".format(exp, tag), test_score, noise_score])
print()
print(tabulate(testScoresTable, headers="firstrow", tablefmt=tablefmt))
def parse_results(config_filename, experiments):
"""
Parse the results for each specified experiment in one cfg file. Creates a
dataframe containing one row per iteration for every trial for every
network configuration in every experiment.
The dataframe is saved to config_filename.pkl The raw results are also saved in a
.csv file named config_filename.csv.
:param config_filename: the cfg filename
:param experiments: a list of experiment names from the cfg file
:return: a dataframe containing raw results
"""
# The results table
columns = [
"Experiment name", "L1 channels", "L2 channels", "L3 N",
"L1 Wt sparsity", "L2 Wt sparsity", "L3 Wt sparsity",
"Activation sparsity", "Non-zero params", "Accuracy", "Iteration",
"Best accuracy", "L2 dimensionality", "L3 dimensionality", "Seed", "ID"
]
df = pd.DataFrame(columns=columns)
# Load and parse experiment configurations
with open(config_filename, "r") as config_file:
configs = parse_config(config_file,
experiments,
globals_param=globals())
for exp in configs:
config = configs[exp]
# Make sure path and data_dir are relative to the project location,
# handling both ~/nta and ../results style paths.
path = config.get("path", ".")
config["path"] = str(Path(path).expanduser().resolve())
data_dir = config.get("data_dir", "data")
config["data_dir"] = str(Path(data_dir).expanduser().resolve())
# Load experiment data
experiment_path = os.path.join(config["path"], exp)
try:
states = load_ray_tune_experiments(experiment_path=experiment_path,
load_results=True)
except RuntimeError:
print("Could not locate experiment state for " + exp +
" ...skipping")
continue
df = parse_one_experiment(exp, states, df)
df.to_csv(os.path.splitext(config_file.name)[0] + ".csv")
df.to_pickle(os.path.splitext(config_file.name)[0] + ".pkl")
return df
def main(config, experiment, tablefmt, show_list):
configs = parse_config(config, experiment, globals_param=globals())
if show_list:
print("Experiments:", list(configs.keys()))
return
params_table = [[
"Network",
"L1 F",
"L1 Sparsity",
"L2 F",
"L2 Sparsity",
"L3 N",
"L3 Sparsity",
"Wt Sparsity",
]]
for name, params in configs.items():
linear_n = params["linear_n"]
linear_percent_on = params["linear_percent_on"]
weight_sparsity = params["weight_sparsity"]
cnn_percent_on = params["cnn_percent_on"]
cnn_out_channels = params["cnn_out_channels"]
l3_n = linear_n[0]
l3_sp = "{0:.1f}%".format(100 * linear_percent_on[0])
wt_sp = "{0}%".format(100 * weight_sparsity)
if len(cnn_percent_on) == 2:
l1_percent_on = cnn_percent_on[0]
l2_percent_on = cnn_percent_on[1]
else:
l1_percent_on = cnn_percent_on[0]
l2_percent_on = None
if len(cnn_out_channels) == 2:
l1_f = cnn_out_channels[0]
l1_sp = "{0:.1f}%".format(100 * l1_percent_on)
# Feed CNN-1 output to CNN-2
l2_f = cnn_out_channels[1]
l2_sp = "{0:.1f}%".format(100 * l2_percent_on)
else:
l1_f = cnn_out_channels[0]
l1_sp = "{0:.1f}%".format(100 * l1_percent_on)
l2_f = None
l2_sp = None
params_table.append(
[name, l1_f, l1_sp, l2_f, l2_sp, l3_n, l3_sp, wt_sp])
print()
print(tabulate(params_table, headers="firstrow", tablefmt=tablefmt))
def plot(config, experiments):
print("config =", config.name)
print("experiments =", experiments)
# Use configuration file location as the project location.
project_dir = os.path.dirname(config.name)
project_dir = os.path.abspath(project_dir)
print("project_dir =", project_dir)
# Load and parse experiment configurations
noise_experiments = defaultdict(list)
configs = parse_config(config, experiments, globals(), locals())
for exp in configs:
config = configs[exp]
# Load experiment state and get all the tags
experiment_path = os.path.join(project_dir, config["path"], exp)
experiment_state = load_ray_tune_experiment(experiment_path)
all_experiments = experiment_state["checkpoints"]
for experiment in all_experiments:
noise_experiments[exp].append(experiment["experiment_tag"])
# Plot noise experiments
for exp in noise_experiments:
fig, ax = plt.subplots()
for tag in noise_experiments[exp]:
# Load experiment results
experiment_path = os.path.join(project_dir, "results", "noise",
exp, tag)
if not os.path.exists(experiment_path):
continue
experiment_state = load_ray_tune_experiment(experiment_path,
load_results=True)
all_experiments = experiment_state["checkpoints"]
data = {}
for experiment in all_experiments:
acc = experiment["results"][0]["mean_accuracy"]
noise = experiment["config"]["noise"]
data.setdefault(noise, acc)
data = OrderedDict(sorted(data.items(), key=lambda i: i[0]))
ax.plot(list(data.keys()), list(data.values()), label=tag)
fig.suptitle("Accuracy vs noise")
ax.set_xlabel("Noise")
ax.set_ylabel("Accuracy (percent)")
plt.legend()
plt.grid(axis="y")
plot_path = os.path.join(project_dir, "results", "noise",
"{}_noise.pdf".format(exp))
plt.savefig(plot_path)
plt.close()
def main(config, experiments):
options = parse_config(config, experiments)
for exp in options:
print("Experiment:", exp)
params = options[exp]
params["name"] = exp
params["loss_function"] = eval(params["loss_function"], globals(),
locals())
exp = NotSoDenseExperiment(params)
print(exp.model)
for epoch in range(exp.epochs):
exp.train(epoch)
print(exp.test())
def noise(config, experiments, num_cpus, num_gpus, redis_address):
print("config =", config.name)
print("num_gpus =", num_gpus)
print("num_cpus =", num_cpus)
print("redis_address =", redis_address)
# Use configuration file location as the project location.
project_dir = os.path.dirname(config.name)
project_dir = os.path.abspath(project_dir)
print("project_dir =", project_dir)
# Load and parse experiment configurations
configs = parse_config(config, experiments, globals_param=globals())
print("experiments =", list(configs.keys()))
# download dataset
data_dir = os.path.join(project_dir, "data")
datasets.MNIST(data_dir, download=True, train=True)
# Initialize ray cluster
if redis_address is not None:
ray.init(redis_address=redis_address, include_webui=True)
else:
ray.init(num_cpus=num_cpus,
num_gpus=num_gpus,
local_mode=num_cpus == 1)
# Run experiments
results = []
for exp in configs:
config = configs[exp]
config["name"] = exp
# Make sure local directories are relative to the project location
path = config.get("path", None)
if path and not os.path.isabs(path):
config["path"] = os.path.join(project_dir, path)
data_dir = config.get("data_dir", "data")
if not os.path.isabs(data_dir):
config["data_dir"] = os.path.join(project_dir, data_dir)
# Avoid "tune.sample_from"
config["seed"] = 18
# Run each experiment in parallel
results.append(run_noise_test.remote(config))
# Wait until all experiments complete
ray.get(results)
ray.shutdown()
def plot(config, experiments):
print("config =", config.name)
print("experiments =", experiments)
# Use configuration file location as the project location.
projectDir = os.path.dirname(config.name)
projectDir = os.path.abspath(projectDir)
print("projectDir =", projectDir)
# Load and parse experiment configurations
noise_experiments = defaultdict(list)
configs = parse_config(config, experiments, globals(), locals())
for exp in configs:
config = configs[exp]
# Load experiment state and get all the tags
experiment_path = os.path.join(projectDir, config["path"], exp)
experiment_state = load_ray_tune_experiment(experiment_path)
all_experiments = experiment_state["checkpoints"]
for experiment in all_experiments:
noise_experiments[exp].append(experiment["experiment_tag"])
# Plot noise experiments
for exp in noise_experiments:
fig, ax = plt.subplots()
for tag in noise_experiments[exp]:
# Load experiment results
experiment_path = os.path.join(projectDir, "results", "noise", exp, tag)
if not os.path.exists(experiment_path):
continue
experiment_state = load_ray_tune_experiment(experiment_path, load_results=True)
all_experiments = experiment_state["checkpoints"]
data = {}
for experiment in all_experiments:
acc = experiment["results"][0]["mean_accuracy"]
noise = experiment["config"]["noise"]
data.setdefault(noise, acc)
data = OrderedDict(sorted(data.items(), key=lambda i: i[0]))
ax.plot(list(data.keys()), list(data.values()), label=tag)
fig.suptitle("Accuracy vs noise")
ax.set_xlabel("Noise")
ax.set_ylabel("Accuracy (percent)")
plt.legend()
plt.grid(axis='y')
plot_path = os.path.join(projectDir, "results", "noise", "{}_noise.pdf".format(exp))
plt.savefig(plot_path)
plt.close()
def noise(config, experiments, num_cpus, num_gpus, redis_address):
print("config =", config.name)
print("num_gpus =", num_gpus)
print("num_cpus =", num_cpus)
print("redis_address =", redis_address)
# Use configuration file location as the project location.
project_dir = os.path.dirname(config.name)
project_dir = os.path.abspath(project_dir)
print("projectDir =", project_dir)
# Load and parse experiment configurations
configs = parse_config(config, experiments, globals=globals())
# Initialize ray cluster
if redis_address is not None:
ray.init(redis_address=redis_address, include_webui=True)
else:
ray.init(num_cpus=num_cpus, num_gpus=num_gpus, local_mode=num_cpus == 1)
# FIXME: Update remote function resource usage
num_gpus = float(num_gpus / num_cpus)
run_noise_test._num_gpus = num_gpus
run_noise_test.num_cpus = 1
# Run experiments
results = []
for exp in configs:
config = configs[exp]
config["name"] = exp
# Make sure local directories are relative to the project location
path = config.get("path", None)
if path and not os.path.isabs(path):
config["path"] = os.path.join(project_dir, path)
data_dir = config.get("data_dir", "data")
if not os.path.isabs(data_dir):
config["data_dir"] = os.path.join(project_dir, data_dir)
# Run each experiment in parallel
results.append(run_noise_test.remote(config))
# Wait until all experiments complete
ray.get(results)
ray.shutdown()
def noise(config, experiments, num_cpus, num_gpus, redis_address):
print("config =", config.name)
print("num_gpus =", num_gpus)
print("num_cpus =", num_cpus)
print("redis_address =", redis_address)
# Use configuration file location as the project location.
project_dir = os.path.dirname(config.name)
project_dir = os.path.abspath(project_dir)
print("project_dir =", project_dir)
# Load and parse experiment configurations
configs = parse_config(config, experiments, globals_param=globals())
# Initialize ray cluster
if redis_address is not None:
ray.init(redis_address=redis_address, include_webui=True)
else:
ray.init(num_cpus=num_cpus, num_gpus=num_gpus, local_mode=num_cpus == 1)
# FIXME: Update remote function resource usage
num_gpus = float(num_gpus / num_cpus)
run_noise_test._num_gpus = num_gpus
run_noise_test.num_cpus = 1
# Run experiments
results = []
for exp in configs:
config = configs[exp]
config["name"] = exp
# Make sure local directories are relative to the project location
path = config.get("path", None)
if path and not os.path.isabs(path):
config["path"] = os.path.join(project_dir, path)
data_dir = config.get("data_dir", "data")
if not os.path.isabs(data_dir):
config["data_dir"] = os.path.join(project_dir, data_dir)
# Run each experiment in parallel
results.append(run_noise_test.remote(config))
# Wait until all experiments complete
ray.get(results)
ray.shutdown()
def main(config, experiment, tablefmt, show_list):
configs = parse_config(config, experiment, globals=globals())
if show_list:
print("Experiments:", list(configs.keys()))
return
params_table = [[
"Network", "L1 F", "L1 Sparsity", "L2 F", "L2 Sparsity", "L3 N",
"L3 Sparsity", "Wt Sparsity"
]]
for name, params in configs.items():
linear_n = params["linear_n"]
linear_percent_on = params["linear_percent_on"]
weight_sparsity = params["weight_sparsity"]
cnn_percent_on = params["cnn_percent_on"]
cnn_out_channels = params["cnn_out_channels"]
l3_n = linear_n[0]
l3_sp = "{0:.1f}%".format(100 * linear_percent_on[0])
wt_sp = "{0}%".format(100 * weight_sparsity)
if len(cnn_percent_on) == 2:
l1_percent_on = cnn_percent_on[0]
l2_percent_on = cnn_percent_on[1]
else:
l1_percent_on = cnn_percent_on[0]
l2_percent_on = None
if len(cnn_out_channels) == 2:
l1_f = cnn_out_channels[0]
l1_sp = "{0:.1f}%".format(100 * l1_percent_on)
# Feed CNN-1 output to CNN-2
l2_f = cnn_out_channels[1]
l2_sp = "{0:.1f}%".format(100 * l2_percent_on)
else:
l1_f = cnn_out_channels[0]
l1_sp = "{0:.1f}%".format(100 * l1_percent_on)
l2_f = None
l2_sp = None
params_table.append([name, l1_f, l1_sp, l2_f, l2_sp, l3_n, l3_sp, wt_sp])
print()
print(tabulate(params_table, headers="firstrow", tablefmt=tablefmt))
def main(config):
# Use configuration file location as the project location.
project_dir = Path(dirname(config.name)).expanduser().resolve()
data_dir = Path(project_dir) / "data"
# Load and parse experiment configurations
configs = parse_config(
config_file=config,
experiments=list(EXPERIMENTS.keys()),
globals_param=globals(),
)
results = {}
for exp in configs:
config = configs[exp]
# Load experiment data
data_dir = Path(config["data_dir"]).expanduser().resolve()
path = Path(config["path"]).expanduser().resolve()
experiment_path = path / exp
experiment_state = load_ray_tune_experiment(
experiment_path=experiment_path, load_results=True)
# Load noise score and compute the mean_accuracy over all checkpoints
exp_df = pd.DataFrame()
for checkpoint in experiment_state["checkpoints"]:
logdir = experiment_path / basename(checkpoint["logdir"])
filename = logdir / "noise.json"
with open(filename, "r") as f:
df = pd.DataFrame(json.load(f)).transpose()
exp_df = exp_df.append(df["mean_accuracy"], ignore_index=True)
results[exp] = exp_df.mean()
plot_path = project_dir / "accuracy_vs_noise.pdf"
plot_noise_curve(configs=configs, results=results, plot_path=plot_path)
# Plot noisy images
plot_path = project_dir / "mnist_images_with_noise.pdf"
plot_images_with_noise(
datadir=data_dir,
noise_values=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5],
plot_path=plot_path,
)
def noise(config, experiments, num_cpus, num_gpus, redis_address):
print("config =", config.name)
print("num_gpus =", num_gpus)
print("num_cpus =", num_cpus)
print("redis_address =", redis_address)
# Load and parse experiment configurations
configs = parse_config(config, experiments, globals_param=globals())
# Initialize ray cluster
if redis_address is not None:
ray.init(redis_address=redis_address, include_webui=True)
else:
ray.init(num_cpus=num_cpus,
num_gpus=num_gpus,
local_mode=num_cpus == 1)
# FIXME: Update remote function resource usage
num_gpus = float(num_gpus / num_cpus)
run_noise_test._num_gpus = num_gpus
run_noise_test.num_cpus = 1
# Run experiments
results = []
for exp in configs:
config = configs[exp]
config["name"] = exp
# Make sure path and data_dir are relative to the project location,
# handling both ~/nta and ../results style paths.
path = config.get("path", ".")
config["path"] = str(Path(path).expanduser().resolve())
data_dir = config.get("data_dir", "data")
config["data_dir"] = str(Path(data_dir).expanduser().resolve())
# Run each experiment in parallel
results.append(run_noise_test.remote(config))
# Wait until all experiments complete
ray.get(results)
ray.shutdown()
print(results)
def main(config):
# Use configuration file location as the project location.
project_dir = os.path.dirname(config.name)
project_dir = os.path.abspath(project_dir)
# Plot noisy images
data_dir = os.path.join(project_dir, "data")
plot_path = os.path.join(project_dir, "mnist_images_with_noise.pdf")
plot_images_with_noise(
datadir=data_dir,
noise_values=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5],
plot_path=plot_path,
)
# Load and parse experiment configurations
configs = parse_config(
config_file=config,
experiments=list(EXPERIMENTS.keys()),
globals_param=globals(),
)
results = {}
for exp in configs:
config = configs[exp]
# Load experiment data
experiment_path = os.path.join(project_dir, config["path"], exp)
experiment_state = load_ray_tune_experiment(
experiment_path=experiment_path, load_results=True)
# Load noise score from the first checkpoint
checkpoint = experiment_state["checkpoints"][0]
logdir = os.path.join(experiment_path,
os.path.basename(checkpoint["logdir"]))
filename = os.path.join(logdir, "noise.json")
with open(filename, "r") as f:
results[exp] = json.load(f)
plot_path = os.path.join(project_dir, "accuracy_vs_noise.pdf")
plot_noise_curve(configs=configs, results=results, plot_path=plot_path)
import numpy as np
import torch
import torch.nn.functional as F
from torch import nn
from cont_speech_experiment import ContinuousSpeechExperiment
from nupic.research.frameworks.continuous_learning.dendrite_layers import DendriteLayer
from nupic.research.frameworks.continuous_learning.utils import freeze_output_layer
from nupic.research.frameworks.pytorch.model_utils import evaluate_model
from nupic.research.support import parse_config
from nupic.torch.modules import Flatten, KWinners2d, SparseWeights, SparseWeights2d
config_file = "experiments.cfg"
with open(config_file) as cf:
config_init = parse_config(cf)
exp = "sparseCNN2"
# play with the config values here
config = config_init[exp]
config["name"] = exp
config["use_dendrites"] = True
config["use_batch_norm"] = False
config["cnn_out_channels"] = (64, 64)
config["cnn_percent_on"] = (0.12, 0.07)
config["cnn_weight_sparsity"] = (0.15, 0.05)
config["dendrites_per_cell"] = 2
config["batch_size"] = 64
def train(config, experiments, num_cpus, num_gpus, redis_address, show_list):
print("config =", config.name)
print("num_gpus =", num_gpus)
print("num_cpus =", num_cpus)
print("redis_address =", redis_address)
# Use configuration file location as the project location.
project_dir = os.path.dirname(config.name)
project_dir = os.path.abspath(project_dir)
print("project_dir =", project_dir)
# Load and parse experiment configurations
configs = parse_config(config, experiments, globals_param=globals())
if show_list:
print("Experiments:", list(configs.keys()))
return
# Initialize ray cluster
if redis_address is not None:
ray.init(redis_address=redis_address, include_webui=True)
num_cpus = 1
else:
ray.init(num_cpus=num_cpus, num_gpus=num_gpus, local_mode=num_cpus == 1)
# Run experiments
resources_per_trial = {"cpu": 1, "gpu": num_gpus / num_cpus}
print("resources_per_trial =", resources_per_trial)
for exp in configs:
print("experiment =", exp)
config = configs[exp]
config["name"] = exp
# Stop criteria. Default to total number of iterations/epochs
stop_criteria = {"training_iteration": config.get("iterations")}
stop_criteria.update(config.get("stop", {}))
print("stop_criteria =", stop_criteria)
# Make sure local directories are relative to the project location
path = config.get("path", None)
if path and not os.path.isabs(path):
config["path"] = os.path.join(project_dir, path)
data_dir = config.get("data_dir", "data")
if not os.path.isabs(data_dir):
config["data_dir"] = os.path.join(project_dir, data_dir)
tune.run(
SpeechExperimentTune,
name=config["name"],
stop=stop_criteria,
config=config,
resources_per_trial=resources_per_trial,
num_samples=config.get("repetitions", 1),
local_dir=config.get("path", None),
upload_dir=config.get("upload_dir", None),
sync_function=config.get("sync_function", None),
checkpoint_freq=config.get("checkpoint_freq", 0),
checkpoint_at_end=config.get("checkpoint_at_end", False),
export_formats=config.get("", None),
search_alg=config.get("search_alg", None),
scheduler=config.get("scheduler", None),
verbose=config.get("verbose", 2),
resume=config.get("resume", False),
queue_trials=config.get("queue_trials", False),
reuse_actors=config.get("reuse_actors", False),
trial_executor=config.get("trial_executor", None),
raise_on_failed_trial=config.get("raise_on_failed_trial", True),
)
ray.shutdown()
def main(config, experiment, tablefmt, show_list):
assert len(experiment) == 2, "Select 2 experiments (denseCNN2, sparseCNN2)"
configs = parse_config(config, experiment, globals_param=globals())
if show_list:
print("Experiments:", list(configs.keys()))
return
# Sort with dense configurations first
configs = sorted(configs.items(),
key=lambda x: 0 if x[0].lower().startswith("dense") else 1)
params_table = [
[
"Network",
"L1",
"L2",
"L3",
"Output",
"Total"
]
]
l1_ratio = l2_ratio = l3_ratio = output_ratio = total_ratio = 1.0
for name, params in configs:
input_shape = params["input_shape"]
input_c, height, width = input_shape
# CNN configuration
cnn_out_channels = params["cnn_out_channels"]
cnn_percent_on = params["cnn_percent_on"]
cnn_weight_sparsity = params["cnn_weight_sparsity"]
# Linear configuration
linear_n = params["linear_n"]
linear_percent_on = params["linear_percent_on"]
weight_sparsity = params["weight_sparsity"]
# Compute total non-zero weights in L1
l1_out_c = cnn_out_channels[0]
l1_w = input_c * l1_out_c * KERNEL_SIZE * KERNEL_SIZE
l1_w = l1_w * cnn_weight_sparsity[0]
# Input density is 1, so L1 multiplies = output_shape * L1 weights
l1_out_width = (width - KERNEL_SIZE + 1)
l1_out_height = (height - KERNEL_SIZE + 1)
l1_mul = l1_out_width * l1_out_height * l1_w
# L1 Output after maxpool
l1_out = [l1_out_c, l1_out_height / 2, l1_out_width / 2]
# Compute total non-zero weights in L2
l2_out_c = cnn_out_channels[1]
l2_w = l1_out_c * l2_out_c * KERNEL_SIZE * KERNEL_SIZE
l2_w = l2_w * cnn_weight_sparsity[1]
# L2 multiplies = input_sparsity * L2 weights * L2 output size
l2_out_height = (l1_out[1] - KERNEL_SIZE + 1)
l2_out_width = (l1_out[2] - KERNEL_SIZE + 1)
l2_mul = cnn_percent_on[0] * l2_out_height * l2_out_width * l2_w
# L2 Output after pool
l2_out = [l2_out_c, l2_out_height / 2, l2_out_width / 2]
# Compute total non-zero weights in L3
l3_w = np.prod(l2_out) * linear_n[0]
l3_w = l3_w * weight_sparsity[0]
# L3 multiplies = l2 sparsity * L3 weights
l3_mul = cnn_percent_on[1] * l3_w
# L3 Output
l3_out = linear_n[0]
l3_nnz_out = l3_out * linear_percent_on[0]
# Output layer multiplies = l3 non-zero output * weights
output_w = l3_out * params["num_classes"]
output_mul = l3_nnz_out * output_w
# Compute gain ratio against previous configuration
l1_ratio = l1_mul / l1_ratio
l2_ratio = l2_mul / l2_ratio
l3_ratio = l3_mul / l3_ratio
output_ratio = output_mul / output_ratio
total_mul = l1_mul + l2_mul + l3_mul + output_mul
total_ratio = total_mul / total_ratio
params_table.append([name,
"{:,.0f}".format(l1_mul),
"{:,.0f}".format(l2_mul),
"{:,.0f}".format(l3_mul),
"{:,.0f}".format(output_mul),
"{:,.0f}".format(total_mul),
])
params_table.append(["Computation Efficiency",
"{:.0f} x".format(1.0 / l1_ratio),
"{:.0f} x".format(1.0 / l2_ratio),
"{:.0f} x".format(1.0 / l3_ratio),
"{:.0f} x".format(1 / output_ratio),
"{:.0f} x".format(1 / total_ratio),
])
print(tabulate(params_table, headers="firstrow", tablefmt=tablefmt,
stralign="center", floatfmt=",.0f"))
def config_init(self, exp):
with open(self.config_file) as cf:
config = parse_config(cf)
exp_config = config[exp]
exp_config["name"] = exp
return exp_config
def train(config, experiments, num_cpus, num_gpus, redis_address, show_list):
print("config =", config.name)
print("num_gpus =", num_gpus)
print("num_cpus =", num_cpus)
print("redis_address =", redis_address)
# Use configuration file location as the project location.
project_dir = os.path.dirname(config.name)
project_dir = os.path.abspath(project_dir)
print("project_dir =", project_dir)
# Load and parse experiment configurations
configs = parse_config(config, experiments, globals_param=globals())
if show_list:
print("Experiments:", list(configs.keys()))
return
print("experiments =", list(configs.keys()))
# Initialize ray cluster
if redis_address is not None:
ray.init(redis_address=redis_address, include_webui=True)
else:
ray.init(num_cpus=num_cpus,
num_gpus=num_gpus,
local_mode=num_cpus == 1)
# Run experiments
gpu_percent = 0
if num_gpus > 0:
gpu_percent = configs.get("gpu_percentage", 0.5)
resources_per_trial = {"cpu": 1, "gpu": gpu_percent}
print("resources_per_trial =", resources_per_trial)
for exp in configs:
print("experiment =", exp)
config = configs[exp]
config["name"] = exp
# Stop criteria. Default to total number of iterations/epochs
stop_criteria = {"training_iteration": config.get("iterations")}
stop_criteria.update(config.get("stop", {}))
print("stop_criteria =", stop_criteria)
# Make sure path and data_dir are relative to the project location,
# handling both ~/nta and ../results style paths.
path = config.get("path", ".")
config["path"] = str(Path(path).expanduser().resolve())
data_dir = config.get("data_dir", "data")
config["data_dir"] = str(Path(data_dir).expanduser().resolve())
tune.run(
MNISTExperimentTune,
name=config["name"],
stop=stop_criteria,
config=config,
resources_per_trial=resources_per_trial,
num_samples=config.get("repetitions", 1),
local_dir=config.get("path", None),
upload_dir=config.get("upload_dir", None),
sync_function=config.get("sync_function", None),
checkpoint_freq=config.get("checkpoint_freq", 0),
checkpoint_at_end=config.get("checkpoint_at_end", False),
export_formats=config.get("", None),
search_alg=config.get("search_alg", None),
scheduler=config.get("scheduler", None),
verbose=config.get("verbose", 2),
resume=config.get("resume", False),
queue_trials=config.get("queue_trials", False),
reuse_actors=config.get("reuse_actors", False),
trial_executor=config.get("trial_executor", None),
raise_on_failed_trial=config.get("raise_on_failed_trial", True),
keep_checkpoints_num=1,
checkpoint_score_attr="mean_accuracy",
)
ray.shutdown()
def main(config, experiments, tablefmt):
# Use configuration file location as the project location.
project_dir = os.path.dirname(config.name)
project_dir = os.path.abspath(project_dir)
print("project_dir =", project_dir)
test_scores_table = [["Network", "Test Score", "Noise Score"]]
# Load and parse experiment configurations
configs = parse_config(config, experiments, globals_param=globals())
# Select tags ignoring seed
def key_func(x):
re.split("[,_]", re.sub(",|\\d+_|seed=\\d+", "", x["experiment_tag"]))
for exp in configs:
config = configs[exp]
# Load experiment data
experiment_path = os.path.join(project_dir, config["path"], exp)
experiment_state = load_ray_tune_experiment(
experiment_path=experiment_path, load_results=True)
# Go through all checkpoints in the experiment
all_checkpoints = experiment_state["checkpoints"]
# Group checkpoints by tags
checkpoint_groups = {
k[0]: list(v)
for k, v in groupby(sorted(all_checkpoints, key=key_func),
key=key_func)
}
for tag in checkpoint_groups:
checkpoints = checkpoint_groups[tag]
num_exps = len(checkpoints)
test_scores = np.zeros(num_exps)
noise_scores = np.zeros(num_exps)
for i, checkpoint in enumerate(checkpoints):
results = checkpoint["results"]
if results is None:
continue
# For each checkpoint select the epoch with the best accuracy as
# the best epoch
best_result = max(results, key=lambda x: x["mean_accuracy"])
test_scores[i] = best_result["mean_accuracy"] * 100.0
# Load noise score
logdir = os.path.join(experiment_path,
os.path.basename(checkpoint["logdir"]))
filename = os.path.join(logdir, "noise.json")
with open(filename, "r") as f:
noise = json.load(f)
noise_scores[i] = sum(x["total_correct"]
for x in list(noise.values()))
test_score = "{0:.2f} ± {1:.2f}".format(test_scores.mean(),
test_scores.std())
noise_score = "{0:,.0f} ± {1:.2f}".format(noise_scores.mean(),
noise_scores.std())
test_scores_table.append(
["{} {}".format(exp, tag), test_score, noise_score])
print()
print(tabulate(test_scores_table, headers="firstrow", tablefmt=tablefmt))
def run(config, experiments, num_cpus, num_gpus, redis_address, noise_values):
print("config =", config.name)
print("experiments =", experiments)
print("num_gpus =", num_gpus)
print("num_cpus =", num_cpus)
print("redis_address =", redis_address)
print("noise_values =", noise_values)
# Use configuration file location as the project location.
project_dir = os.path.dirname(config.name)
project_dir = os.path.abspath(project_dir)
print("project_dir =", project_dir)
# Download dataset
data_dir = os.path.join(project_dir, "data")
datasets.CIFAR10(data_dir, download=True, train=True)
# Initialize ray cluster
if redis_address is not None:
ray.init(redis_address=redis_address, include_webui=True)
else:
ray.init(num_cpus=num_cpus,
num_gpus=num_gpus,
local_mode=num_cpus == 1)
# Load and parse experiment configurations
configs = parse_config(config, experiments, globals(), locals())
# Run all experiments in parallel
ray_trials = []
for exp in configs:
config = configs[exp]
# noise experiment tune configuration
noise_config = {
"iterations": 1,
"noise": {
"grid_search": list(noise_values)
}
}
# Download results from S3 when running on the cluster
if redis_address is not None and "upload_dir" in config:
upload_dir = config["upload_dir"]
download_s3_results("{}/{}".format(upload_dir, exp),
os.path.join(config["path"], exp))
# Store noise results with original results in S3
noise_config["upload_dir"] = "{}".format(upload_dir)
noise_config[
"sync_function"] = "aws s3 sync `dirname {local_dir}` {remote_dir}/`basename $(dirname {local_dir})`" # noqa E501
else:
noise_config.pop("upload_dir", None)
noise_config.pop("sync_function", None)
# Load experiment results
experiment_path = os.path.join(project_dir, config["path"], exp)
experiment_state = load_ray_tune_experiment(experiment_path,
load_results=True)
all_experiments = experiment_state["checkpoints"]
for experiment in all_experiments:
# Make logs relative to experiment path
logdir = experiment["logdir"]
logpath = os.path.join(experiment_path, os.path.basename(logdir))
# Check for experiment results
results = experiment["results"]
if results is None:
continue
# Get best scoring model checkpoint from results
best_result = max(results, key=lambda x: x["mean_accuracy"])
epoch = best_result["training_iteration"]
checkpoint_path = os.path.join(logpath,
"checkpoint_{}".format(epoch))
if os.path.exists(checkpoint_path):
# Update data path
model_config = best_result["config"]
model_config["data_dir"] = data_dir
# Run noise tests
noise_config.update({
"name":
experiment["experiment_tag"],
"path":
os.path.join(project_dir, "results", "noise", exp),
"checkpoint_path":
checkpoint_path,
"model_config":
model_config,
})
ray_trials.append(
run_experiment.remote(
noise_config,
MobileNetNoiseTune,
num_cpus=1,
num_gpus=min(1, num_gpus),
))
# Wait for all experiments to complete
ray.get(ray_trials)
ray.shutdown()
def train(config, experiments, num_cpus, num_gpus, redis_address, show_list):
print("config =", config.name)
print("num_gpus =", num_gpus)
print("num_cpus =", num_cpus)
print("redis_address =", redis_address)
# Use configuration file location as the project location.
project_dir = os.path.dirname(config.name)
project_dir = os.path.abspath(project_dir)
print("project_dir =", project_dir)
# Load and parse experiment configurations
configs = parse_config(config, experiments, globals=globals())
if show_list:
print("Experiments:", list(configs.keys()))
return
# Initialize ray cluster
if redis_address is not None:
ray.init(redis_address=redis_address, include_webui=True)
num_cpus = 1
else:
ray.init(num_cpus=num_cpus, num_gpus=num_gpus, local_mode=num_cpus == 1)
# Run experiments
resources_per_trial = {"cpu": 1, "gpu": num_gpus / num_cpus}
print("resources_per_trial =", resources_per_trial)
for exp in configs:
print("experiment =", exp)
config = configs[exp]
config["name"] = exp
# Stop criteria. Default to total number of iterations/epochs
stop_criteria = {"training_iteration": config.get("iterations")}
stop_criteria.update(config.get("stop", {}))
print("stop_criteria =", stop_criteria)
# Make sure local directories are relative to the project location
path = config.get("path", None)
if path and not os.path.isabs(path):
config["path"] = os.path.join(project_dir, path)
data_dir = config.get("data_dir", "data")
if not os.path.isabs(data_dir):
config["data_dir"] = os.path.join(project_dir, data_dir)
tune.run(
SpeechExperimentTune,
name=config["name"],
stop=stop_criteria,
config=config,
resources_per_trial=resources_per_trial,
num_samples=config.get("repetitions", 1),
local_dir=config.get("path", None),
upload_dir=config.get("upload_dir", None),
sync_function=config.get("sync_function", None),
checkpoint_freq=config.get("checkpoint_freq", 0),
checkpoint_at_end=config.get("checkpoint_at_end", False),
export_formats=config.get("", None),
search_alg=config.get("search_alg", None),
scheduler=config.get("scheduler", None),
verbose=config.get("verbose", 2),
resume=config.get("resume", False),
queue_trials=config.get("queue_trials", False),
reuse_actors=config.get("reuse_actors", False),
trial_executor=config.get("trial_executor", None),
raise_on_failed_trial=config.get("raise_on_failed_trial", True)
)
ray.shutdown()
def main(config, experiments, tablefmt):
# The table we use in the paper
test_scores_table = [["Network", "Test Score", "Noise Score", "Params"]]
# A more detailed table
test_scores_table_long = [[
"Network", "Test Score", "Noise Score", "Noise Accuracy",
"Total Entropy", "Nonzero Parameters", "Num Trials", "Session"
]]
# Load and parse experiment configurations
configs = parse_config(config, experiments, globals_param=globals())
# Use the appropriate plus/minus sign for latex
if tablefmt == "grid":
pm = "±"
else:
pm = "$\\pm$"
# Select tags ignoring seed value
def key_func(x):
s = re.split("[,]", re.sub(",|\\d+_|seed=\\d+", "",
x["experiment_tag"]))
if len(s[0]) == 0:
return [" "]
return s
for exp in configs:
config = configs[exp]
# Make sure path and data_dir are relative to the project location,
# handling both ~/nta and ../results style paths.
path = config.get("path", ".")
config["path"] = str(Path(path).expanduser().resolve())
data_dir = config.get("data_dir", "data")
config["data_dir"] = str(Path(data_dir).expanduser().resolve())
# Load experiment data
experiment_path = os.path.join(config["path"], exp)
try:
states = load_ray_tune_experiments(experiment_path=experiment_path,
load_results=True)
except RuntimeError:
# print("Could not locate experiment state for " + exp + " ...skipping")
continue
for experiment_state in states:
# Go through all checkpoints in the experiment
all_checkpoints = experiment_state["checkpoints"]
# Group checkpoints by tags
checkpoint_groups = {
k[0]: list(v)
for k, v in groupby(sorted(all_checkpoints, key=key_func),
key=key_func)
}
for tag in checkpoint_groups:
checkpoints = checkpoint_groups[tag]
num_exps = len(checkpoints)
test_scores = np.zeros(num_exps)
noise_scores = np.zeros(num_exps)
noise_accuracies = np.zeros(num_exps)
noise_samples = np.zeros(num_exps)
nonzero_params = np.zeros(num_exps)
entropies = np.zeros(num_exps)
try:
for i, checkpoint in enumerate(checkpoints):
results = checkpoint["results"]
if results is None:
continue
# For each checkpoint select the epoch with the best accuracy as
# the best epoch
best_result = max(results,
key=lambda x: x["mean_accuracy"])
test_scores[i] = best_result["mean_accuracy"]
entropies[i] = best_result["entropy"]
# print("best result:", best_result)
# Load noise score
logdir = os.path.join(
experiment_path,
os.path.basename(checkpoint["logdir"]))
filename = os.path.join(logdir, "noise.json")
if os.path.exists(filename):
with open(filename, "r") as f:
noise = json.load(f)
noise_scores[i] = sum(
x["total_correct"]
for x in list(noise.values()))
noise_samples[i] = sum(
x["total_samples"]
for x in list(noise.values()))
else:
print("No noise file for " + experiment_path +
" ...skipping")
continue
noise_accuracies[i] = (float(100.0 * noise_scores[i]) /
noise_samples[i])
nonzero_params[i] = max(x["non_zero_parameters"]
for x in list(noise.values()))
except Exception:
print("Problem with checkpoint group" + tag + " in " +
exp + " ...skipping")
continue
test_score = "{0:.2f} {1:} {2:.2f}".format(
test_scores.mean(), pm, test_scores.std())
entropy = "{0:.2f} {1:} {2:.2f}".format(
entropies.mean(), pm, entropies.std())
noise_score = "{0:,.0f} {1:} {2:.2f}".format(
noise_scores.mean(), pm, noise_scores.std())
noise_accuracy = "{0:,.2f} {1:} {2:.2f}".format(
noise_accuracies.mean(), pm, noise_accuracies.std())
nonzero = "{0:,.0f}".format(nonzero_params.mean())
test_scores_table.append([
"{} {}".format(exp, tag), test_score, noise_accuracy,
nonzero
])
test_scores_table_long.append([
"{} {}".format(exp, tag), test_score, noise_score,
noise_accuracy, entropy, nonzero, num_exps,
experiment_state["runner_data"]["_session_str"]
])
print()
print(tabulate(test_scores_table, headers="firstrow", tablefmt=tablefmt))
print()
print(
tabulate(test_scores_table_long, headers="firstrow",
tablefmt=tablefmt))
def run(config, experiments, num_cpus, num_gpus, redis_address, noise_values):
print("config =", config.name)
print("experiments =", experiments)
print("num_gpus =", num_gpus)
print("num_cpus =", num_cpus)
print("redis_address =", redis_address)
print("noise_values =", noise_values)
# Use configuration file location as the project location.
projectDir = os.path.dirname(config.name)
projectDir = os.path.abspath(projectDir)
print("projectDir =", projectDir)
# Download dataset
data_dir = os.path.join(projectDir, "data")
datasets.CIFAR10(data_dir, download=True, train=True)
# Initialize ray cluster
if redis_address is not None:
ray.init(redis_address=redis_address, include_webui=True)
else:
ray.init(num_cpus=num_cpus, num_gpus=num_gpus, local_mode=num_cpus == 1)
# Load and parse experiment configurations
configs = parse_config(config, experiments, globals(), locals())
# Run all experiments in parallel
ray_trials = []
for exp in configs:
config = configs[exp]
# noise experiment tune configuration
noise_config = {
"iterations": 1,
"noise": {"grid_search": list(noise_values)}
}
# Download results from S3 when running on the cluster
if redis_address is not None and "upload_dir" in config:
upload_dir = config["upload_dir"]
download_s3_results("{}/{}".format(upload_dir, exp),
os.path.join(config["path"], exp))
# Store noise results with original results in S3
noise_config["upload_dir"] = "{}".format(upload_dir)
noise_config["sync_function"] = "aws s3 sync `dirname {local_dir}` {remote_dir}/`basename $(dirname {local_dir})`"
else:
noise_config.pop("upload_dir", None)
noise_config.pop("sync_function", None)
# Load experiment results
experiment_path = os.path.join(projectDir, config["path"], exp)
experiment_state = load_ray_tune_experiment(experiment_path,
load_results=True)
all_experiments = experiment_state["checkpoints"]
for experiment in all_experiments:
# Make logs relative to experiment path
logdir = experiment["logdir"]
logpath = os.path.join(experiment_path, os.path.basename(logdir))
# Check for experiment results
results = experiment["results"]
if results is None:
continue
# Get best scoring model checkpoint from results
best_result = max(results, key=lambda x: x["mean_accuracy"])
epoch = best_result["training_iteration"]
checkpoint_path = os.path.join(logpath, "checkpoint_{}".format(epoch))
if os.path.exists(checkpoint_path):
# Update data path
model_config = best_result["config"]
model_config["data_dir"] = data_dir
# Run noise tests
noise_config.update({
"name": experiment["experiment_tag"],
"path": os.path.join(projectDir, "results", "noise", exp),
"checkpoint_path": checkpoint_path,
"model_config": model_config
})
ray_trials.append(
run_experiment.remote(noise_config, MobileNetNoiseTune,
num_cpus=1, num_gpus=min(1, num_gpus)))
# Wait for all experiments to complete
ray.get(ray_trials)
ray.shutdown()
def main(config, experiment, tablefmt, show_list):
configs = parse_config(config, experiment, globals_param=globals())
if show_list:
print("Experiments:", list(configs.keys()))
return
params_table = [[
"Network",
"L1 Filters",
"L1 Act Sparsity",
"L1 Wt Sparsity",
"L2 Filters",
"L2 Act Sparsity",
"L2 Wt Sparsity",
"L3 N",
"L3 Act Sparsity",
"Wt Sparsity",
]]
params_table1 = [[
"Network",
"L1 Channels",
"L2 Channels",
"L3 N",
]]
params_table2 = [[
"Network",
"L1 Activation Sparsity",
"L2 Activation Sparsity",
"L3 Activation Sparsity",
]]
params_table3 = [[
"Network",
"L1 Weight Sparsity",
"L2 Weight Sparsity",
"L3 Weight Sparsity",
]]
for name, params in configs.items():
linear_n = params["linear_n"]
linear_percent_on = params["linear_percent_on"]
weight_sparsity = params["weight_sparsity"]
cnn_percent_on = params["cnn_percent_on"]
cnn_out_channels = params["cnn_out_channels"]
cnn_weight_sparsity = params["cnn_weight_sparsity"]
l3_n = linear_n[0]
if linear_percent_on[0] > 0.50:
l3_sp = "ReLU"
else:
l3_sp = "{0:.1f}%".format(100 * (1.0 - linear_percent_on[0]))
wt_sp = "{0}%".format(100 * (1.0 - weight_sparsity[0]))
l1_percent_on = cnn_percent_on[0]
l1_wt_sparsity = cnn_weight_sparsity[0]
if len(cnn_percent_on) == 2:
l2_percent_on = cnn_percent_on[1]
l2_wt_sparsity = cnn_weight_sparsity[1]
else:
l2_percent_on = None
l2_wt_sparsity = None
l1_f = cnn_out_channels[0]
if l1_percent_on > 0.50:
l1_sp = "ReLU"
else:
l1_sp = "{0:.1f}%".format(100 * (1.0 - l1_percent_on))
l1_wt = "{0:.1f}%".format(100 * (1.0 - l1_wt_sparsity))
if len(cnn_out_channels) == 2:
l2_f = cnn_out_channels[1]
if l2_percent_on > 0.50:
l2_sp = "ReLU"
else:
l2_sp = "{0:.1f}%".format(100 * (1.0 - l2_percent_on))
l2_wt = "{0:.1f}%".format(100 * (1.0 - l2_wt_sparsity))
else:
l2_f = None
l2_sp = None
l2_wt = None
params_table.append(
[name, l1_f, l1_sp, l1_wt, l2_f, l2_sp, l2_wt, l3_n, l3_sp, wt_sp])
params_table1.append([name, l1_f, l2_f, l3_n])
params_table2.append([name, l1_sp, l2_sp, l3_sp])
params_table3.append([name, l1_wt, l2_wt, wt_sp])
print()
print(
tabulate(
params_table,
headers="firstrow",
tablefmt=tablefmt,
stralign="center",
))
print()
print(
tabulate(params_table1,
headers="firstrow",
tablefmt=tablefmt,
stralign="left",
numalign="center"))
print()
print(
tabulate(params_table2,
headers="firstrow",
tablefmt=tablefmt,
stralign="left",
numalign="center"))
print()
print(
tabulate(params_table3,
headers="firstrow",
tablefmt=tablefmt,
stralign="left",
numalign="center"))
