def main(config): # Use configuration file location as the project location. project_dir = os.path.dirname(config.name) project_dir = os.path.abspath(project_dir) # Plot noisy images data_dir = os.path.join(project_dir, "data") plotPath = os.path.join(project_dir, "mnist_images_with_noise.pdf") plotImagesWithNoise(datadir=data_dir, noise_values=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5], plotPath=plotPath) # Load and parse experiment configurations configs = parse_config(config_file=config, experiments=list(EXPERIMENTS.keys()), globals=globals()) results = {} for exp in configs: config = configs[exp] # Load experiment data experiment_path = os.path.join(project_dir, config["path"], exp) experiment_state = load_ray_tune_experiment( experiment_path=experiment_path, load_results=True) # Load noise score from the first checkpoint checkpoint = experiment_state["checkpoints"][0] logdir = os.path.join(experiment_path, os.path.basename(checkpoint["logdir"])) filename = os.path.join(logdir, "noise.json") with open(filename, "r") as f: results[exp] = json.load(f) plotPath = os.path.join(project_dir, "accuracy_vs_noise.pdf") plotNoiseCurve(configs=configs, results=results, plotPath=plotPath)
def main(config, experiments, tablefmt): # Use configuration file location as the project location. project_dir = os.path.dirname(config.name) project_dir = os.path.abspath(project_dir) print("project_dir =", project_dir) testScoresTable = [["Network", "Test Score", "Noise Score"]] # Load and parse experiment configurations configs = parse_config(config, experiments, globals=globals()) # Select tags ignoring seed key_func = lambda x: re.split("[,_]", re.sub(",|\\d+_|seed=\\d+", "", x["experiment_tag"])) for exp in configs: config = configs[exp] # Load experiment data experiment_path = os.path.join(project_dir, config["path"], exp) experiment_state = load_ray_tune_experiment( experiment_path=experiment_path, load_results=True) # Go through all checkpoints in the experiment all_checkpoints = experiment_state["checkpoints"] # Group checkpoints by tags checkpoint_groups = {k[0]: list(v) for k, v in groupby( sorted(all_checkpoints, key=key_func), key=key_func)} for tag in checkpoint_groups: checkpoints = checkpoint_groups[tag] numExps = len(checkpoints) testScores = np.zeros(numExps) noiseScores = np.zeros(numExps) for i, checkpoint in enumerate(checkpoints): results = checkpoint["results"] if results is None: continue # For each checkpoint select the epoch with the best accuracy as the best epoch best_result = max(results, key=lambda x: x["mean_accuracy"]) testScores[i] = best_result["mean_accuracy"] * 100.0 # Load noise score logdir = os.path.join(experiment_path, os.path.basename(checkpoint["logdir"])) filename = os.path.join(logdir, "noise.json") with open(filename, "r") as f: noise = json.load(f) noiseScores[i] = sum([x["total_correct"] for x in list(noise.values())]) test_score = u"{0:.2f} ± {1:.2f}".format(testScores.mean(), testScores.std()) noise_score = u"{0:,.0f} ± {1:.2f}".format(noiseScores.mean(), noiseScores.std()) testScoresTable.append(["{} {}".format(exp, tag), test_score, noise_score]) print() print(tabulate(testScoresTable, headers="firstrow", tablefmt=tablefmt))
def parse_results(config_filename, experiments): """ Parse the results for each specified experiment in one cfg file. Creates a dataframe containing one row per iteration for every trial for every network configuration in every experiment. The dataframe is saved to config_filename.pkl The raw results are also saved in a .csv file named config_filename.csv. :param config_filename: the cfg filename :param experiments: a list of experiment names from the cfg file :return: a dataframe containing raw results """ # The results table columns = [ "Experiment name", "L1 channels", "L2 channels", "L3 N", "L1 Wt sparsity", "L2 Wt sparsity", "L3 Wt sparsity", "Activation sparsity", "Non-zero params", "Accuracy", "Iteration", "Best accuracy", "L2 dimensionality", "L3 dimensionality", "Seed", "ID" ] df = pd.DataFrame(columns=columns) # Load and parse experiment configurations with open(config_filename, "r") as config_file: configs = parse_config(config_file, experiments, globals_param=globals()) for exp in configs: config = configs[exp] # Make sure path and data_dir are relative to the project location, # handling both ~/nta and ../results style paths. path = config.get("path", ".") config["path"] = str(Path(path).expanduser().resolve()) data_dir = config.get("data_dir", "data") config["data_dir"] = str(Path(data_dir).expanduser().resolve()) # Load experiment data experiment_path = os.path.join(config["path"], exp) try: states = load_ray_tune_experiments(experiment_path=experiment_path, load_results=True) except RuntimeError: print("Could not locate experiment state for " + exp + " ...skipping") continue df = parse_one_experiment(exp, states, df) df.to_csv(os.path.splitext(config_file.name)[0] + ".csv") df.to_pickle(os.path.splitext(config_file.name)[0] + ".pkl") return df
def main(config, experiment, tablefmt, show_list): configs = parse_config(config, experiment, globals_param=globals()) if show_list: print("Experiments:", list(configs.keys())) return params_table = [[ "Network", "L1 F", "L1 Sparsity", "L2 F", "L2 Sparsity", "L3 N", "L3 Sparsity", "Wt Sparsity", ]] for name, params in configs.items(): linear_n = params["linear_n"] linear_percent_on = params["linear_percent_on"] weight_sparsity = params["weight_sparsity"] cnn_percent_on = params["cnn_percent_on"] cnn_out_channels = params["cnn_out_channels"] l3_n = linear_n[0] l3_sp = "{0:.1f}%".format(100 * linear_percent_on[0]) wt_sp = "{0}%".format(100 * weight_sparsity) if len(cnn_percent_on) == 2: l1_percent_on = cnn_percent_on[0] l2_percent_on = cnn_percent_on[1] else: l1_percent_on = cnn_percent_on[0] l2_percent_on = None if len(cnn_out_channels) == 2: l1_f = cnn_out_channels[0] l1_sp = "{0:.1f}%".format(100 * l1_percent_on) # Feed CNN-1 output to CNN-2 l2_f = cnn_out_channels[1] l2_sp = "{0:.1f}%".format(100 * l2_percent_on) else: l1_f = cnn_out_channels[0] l1_sp = "{0:.1f}%".format(100 * l1_percent_on) l2_f = None l2_sp = None params_table.append( [name, l1_f, l1_sp, l2_f, l2_sp, l3_n, l3_sp, wt_sp]) print() print(tabulate(params_table, headers="firstrow", tablefmt=tablefmt))
def plot(config, experiments): print("config =", config.name) print("experiments =", experiments) # Use configuration file location as the project location. project_dir = os.path.dirname(config.name) project_dir = os.path.abspath(project_dir) print("project_dir =", project_dir) # Load and parse experiment configurations noise_experiments = defaultdict(list) configs = parse_config(config, experiments, globals(), locals()) for exp in configs: config = configs[exp] # Load experiment state and get all the tags experiment_path = os.path.join(project_dir, config["path"], exp) experiment_state = load_ray_tune_experiment(experiment_path) all_experiments = experiment_state["checkpoints"] for experiment in all_experiments: noise_experiments[exp].append(experiment["experiment_tag"]) # Plot noise experiments for exp in noise_experiments: fig, ax = plt.subplots() for tag in noise_experiments[exp]: # Load experiment results experiment_path = os.path.join(project_dir, "results", "noise", exp, tag) if not os.path.exists(experiment_path): continue experiment_state = load_ray_tune_experiment(experiment_path, load_results=True) all_experiments = experiment_state["checkpoints"] data = {} for experiment in all_experiments: acc = experiment["results"][0]["mean_accuracy"] noise = experiment["config"]["noise"] data.setdefault(noise, acc) data = OrderedDict(sorted(data.items(), key=lambda i: i[0])) ax.plot(list(data.keys()), list(data.values()), label=tag) fig.suptitle("Accuracy vs noise") ax.set_xlabel("Noise") ax.set_ylabel("Accuracy (percent)") plt.legend() plt.grid(axis="y") plot_path = os.path.join(project_dir, "results", "noise", "{}_noise.pdf".format(exp)) plt.savefig(plot_path) plt.close()
def main(config, experiments): options = parse_config(config, experiments) for exp in options: print("Experiment:", exp) params = options[exp] params["name"] = exp params["loss_function"] = eval(params["loss_function"], globals(), locals()) exp = NotSoDenseExperiment(params) print(exp.model) for epoch in range(exp.epochs): exp.train(epoch) print(exp.test())
def noise(config, experiments, num_cpus, num_gpus, redis_address): print("config =", config.name) print("num_gpus =", num_gpus) print("num_cpus =", num_cpus) print("redis_address =", redis_address) # Use configuration file location as the project location. project_dir = os.path.dirname(config.name) project_dir = os.path.abspath(project_dir) print("project_dir =", project_dir) # Load and parse experiment configurations configs = parse_config(config, experiments, globals_param=globals()) print("experiments =", list(configs.keys())) # download dataset data_dir = os.path.join(project_dir, "data") datasets.MNIST(data_dir, download=True, train=True) # Initialize ray cluster if redis_address is not None: ray.init(redis_address=redis_address, include_webui=True) else: ray.init(num_cpus=num_cpus, num_gpus=num_gpus, local_mode=num_cpus == 1) # Run experiments results = [] for exp in configs: config = configs[exp] config["name"] = exp # Make sure local directories are relative to the project location path = config.get("path", None) if path and not os.path.isabs(path): config["path"] = os.path.join(project_dir, path) data_dir = config.get("data_dir", "data") if not os.path.isabs(data_dir): config["data_dir"] = os.path.join(project_dir, data_dir) # Avoid "tune.sample_from" config["seed"] = 18 # Run each experiment in parallel results.append(run_noise_test.remote(config)) # Wait until all experiments complete ray.get(results) ray.shutdown()
def plot(config, experiments): print("config =", config.name) print("experiments =", experiments) # Use configuration file location as the project location. projectDir = os.path.dirname(config.name) projectDir = os.path.abspath(projectDir) print("projectDir =", projectDir) # Load and parse experiment configurations noise_experiments = defaultdict(list) configs = parse_config(config, experiments, globals(), locals()) for exp in configs: config = configs[exp] # Load experiment state and get all the tags experiment_path = os.path.join(projectDir, config["path"], exp) experiment_state = load_ray_tune_experiment(experiment_path) all_experiments = experiment_state["checkpoints"] for experiment in all_experiments: noise_experiments[exp].append(experiment["experiment_tag"]) # Plot noise experiments for exp in noise_experiments: fig, ax = plt.subplots() for tag in noise_experiments[exp]: # Load experiment results experiment_path = os.path.join(projectDir, "results", "noise", exp, tag) if not os.path.exists(experiment_path): continue experiment_state = load_ray_tune_experiment(experiment_path, load_results=True) all_experiments = experiment_state["checkpoints"] data = {} for experiment in all_experiments: acc = experiment["results"][0]["mean_accuracy"] noise = experiment["config"]["noise"] data.setdefault(noise, acc) data = OrderedDict(sorted(data.items(), key=lambda i: i[0])) ax.plot(list(data.keys()), list(data.values()), label=tag) fig.suptitle("Accuracy vs noise") ax.set_xlabel("Noise") ax.set_ylabel("Accuracy (percent)") plt.legend() plt.grid(axis='y') plot_path = os.path.join(projectDir, "results", "noise", "{}_noise.pdf".format(exp)) plt.savefig(plot_path) plt.close()
def noise(config, experiments, num_cpus, num_gpus, redis_address): print("config =", config.name) print("num_gpus =", num_gpus) print("num_cpus =", num_cpus) print("redis_address =", redis_address) # Use configuration file location as the project location. project_dir = os.path.dirname(config.name) project_dir = os.path.abspath(project_dir) print("projectDir =", project_dir) # Load and parse experiment configurations configs = parse_config(config, experiments, globals=globals()) # Initialize ray cluster if redis_address is not None: ray.init(redis_address=redis_address, include_webui=True) else: ray.init(num_cpus=num_cpus, num_gpus=num_gpus, local_mode=num_cpus == 1) # FIXME: Update remote function resource usage num_gpus = float(num_gpus / num_cpus) run_noise_test._num_gpus = num_gpus run_noise_test.num_cpus = 1 # Run experiments results = [] for exp in configs: config = configs[exp] config["name"] = exp # Make sure local directories are relative to the project location path = config.get("path", None) if path and not os.path.isabs(path): config["path"] = os.path.join(project_dir, path) data_dir = config.get("data_dir", "data") if not os.path.isabs(data_dir): config["data_dir"] = os.path.join(project_dir, data_dir) # Run each experiment in parallel results.append(run_noise_test.remote(config)) # Wait until all experiments complete ray.get(results) ray.shutdown()
def noise(config, experiments, num_cpus, num_gpus, redis_address): print("config =", config.name) print("num_gpus =", num_gpus) print("num_cpus =", num_cpus) print("redis_address =", redis_address) # Use configuration file location as the project location. project_dir = os.path.dirname(config.name) project_dir = os.path.abspath(project_dir) print("project_dir =", project_dir) # Load and parse experiment configurations configs = parse_config(config, experiments, globals_param=globals()) # Initialize ray cluster if redis_address is not None: ray.init(redis_address=redis_address, include_webui=True) else: ray.init(num_cpus=num_cpus, num_gpus=num_gpus, local_mode=num_cpus == 1) # FIXME: Update remote function resource usage num_gpus = float(num_gpus / num_cpus) run_noise_test._num_gpus = num_gpus run_noise_test.num_cpus = 1 # Run experiments results = [] for exp in configs: config = configs[exp] config["name"] = exp # Make sure local directories are relative to the project location path = config.get("path", None) if path and not os.path.isabs(path): config["path"] = os.path.join(project_dir, path) data_dir = config.get("data_dir", "data") if not os.path.isabs(data_dir): config["data_dir"] = os.path.join(project_dir, data_dir) # Run each experiment in parallel results.append(run_noise_test.remote(config)) # Wait until all experiments complete ray.get(results) ray.shutdown()
def main(config, experiment, tablefmt, show_list): configs = parse_config(config, experiment, globals=globals()) if show_list: print("Experiments:", list(configs.keys())) return params_table = [[ "Network", "L1 F", "L1 Sparsity", "L2 F", "L2 Sparsity", "L3 N", "L3 Sparsity", "Wt Sparsity" ]] for name, params in configs.items(): linear_n = params["linear_n"] linear_percent_on = params["linear_percent_on"] weight_sparsity = params["weight_sparsity"] cnn_percent_on = params["cnn_percent_on"] cnn_out_channels = params["cnn_out_channels"] l3_n = linear_n[0] l3_sp = "{0:.1f}%".format(100 * linear_percent_on[0]) wt_sp = "{0}%".format(100 * weight_sparsity) if len(cnn_percent_on) == 2: l1_percent_on = cnn_percent_on[0] l2_percent_on = cnn_percent_on[1] else: l1_percent_on = cnn_percent_on[0] l2_percent_on = None if len(cnn_out_channels) == 2: l1_f = cnn_out_channels[0] l1_sp = "{0:.1f}%".format(100 * l1_percent_on) # Feed CNN-1 output to CNN-2 l2_f = cnn_out_channels[1] l2_sp = "{0:.1f}%".format(100 * l2_percent_on) else: l1_f = cnn_out_channels[0] l1_sp = "{0:.1f}%".format(100 * l1_percent_on) l2_f = None l2_sp = None params_table.append([name, l1_f, l1_sp, l2_f, l2_sp, l3_n, l3_sp, wt_sp]) print() print(tabulate(params_table, headers="firstrow", tablefmt=tablefmt))
def main(config): # Use configuration file location as the project location. project_dir = Path(dirname(config.name)).expanduser().resolve() data_dir = Path(project_dir) / "data" # Load and parse experiment configurations configs = parse_config( config_file=config, experiments=list(EXPERIMENTS.keys()), globals_param=globals(), ) results = {} for exp in configs: config = configs[exp] # Load experiment data data_dir = Path(config["data_dir"]).expanduser().resolve() path = Path(config["path"]).expanduser().resolve() experiment_path = path / exp experiment_state = load_ray_tune_experiment( experiment_path=experiment_path, load_results=True) # Load noise score and compute the mean_accuracy over all checkpoints exp_df = pd.DataFrame() for checkpoint in experiment_state["checkpoints"]: logdir = experiment_path / basename(checkpoint["logdir"]) filename = logdir / "noise.json" with open(filename, "r") as f: df = pd.DataFrame(json.load(f)).transpose() exp_df = exp_df.append(df["mean_accuracy"], ignore_index=True) results[exp] = exp_df.mean() plot_path = project_dir / "accuracy_vs_noise.pdf" plot_noise_curve(configs=configs, results=results, plot_path=plot_path) # Plot noisy images plot_path = project_dir / "mnist_images_with_noise.pdf" plot_images_with_noise( datadir=data_dir, noise_values=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5], plot_path=plot_path, )
def noise(config, experiments, num_cpus, num_gpus, redis_address): print("config =", config.name) print("num_gpus =", num_gpus) print("num_cpus =", num_cpus) print("redis_address =", redis_address) # Load and parse experiment configurations configs = parse_config(config, experiments, globals_param=globals()) # Initialize ray cluster if redis_address is not None: ray.init(redis_address=redis_address, include_webui=True) else: ray.init(num_cpus=num_cpus, num_gpus=num_gpus, local_mode=num_cpus == 1) # FIXME: Update remote function resource usage num_gpus = float(num_gpus / num_cpus) run_noise_test._num_gpus = num_gpus run_noise_test.num_cpus = 1 # Run experiments results = [] for exp in configs: config = configs[exp] config["name"] = exp # Make sure path and data_dir are relative to the project location, # handling both ~/nta and ../results style paths. path = config.get("path", ".") config["path"] = str(Path(path).expanduser().resolve()) data_dir = config.get("data_dir", "data") config["data_dir"] = str(Path(data_dir).expanduser().resolve()) # Run each experiment in parallel results.append(run_noise_test.remote(config)) # Wait until all experiments complete ray.get(results) ray.shutdown() print(results)
def main(config): # Use configuration file location as the project location. project_dir = os.path.dirname(config.name) project_dir = os.path.abspath(project_dir) # Plot noisy images data_dir = os.path.join(project_dir, "data") plot_path = os.path.join(project_dir, "mnist_images_with_noise.pdf") plot_images_with_noise( datadir=data_dir, noise_values=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5], plot_path=plot_path, ) # Load and parse experiment configurations configs = parse_config( config_file=config, experiments=list(EXPERIMENTS.keys()), globals_param=globals(), ) results = {} for exp in configs: config = configs[exp] # Load experiment data experiment_path = os.path.join(project_dir, config["path"], exp) experiment_state = load_ray_tune_experiment( experiment_path=experiment_path, load_results=True) # Load noise score from the first checkpoint checkpoint = experiment_state["checkpoints"][0] logdir = os.path.join(experiment_path, os.path.basename(checkpoint["logdir"])) filename = os.path.join(logdir, "noise.json") with open(filename, "r") as f: results[exp] = json.load(f) plot_path = os.path.join(project_dir, "accuracy_vs_noise.pdf") plot_noise_curve(configs=configs, results=results, plot_path=plot_path)
import numpy as np import torch import torch.nn.functional as F from torch import nn from cont_speech_experiment import ContinuousSpeechExperiment from nupic.research.frameworks.continuous_learning.dendrite_layers import DendriteLayer from nupic.research.frameworks.continuous_learning.utils import freeze_output_layer from nupic.research.frameworks.pytorch.model_utils import evaluate_model from nupic.research.support import parse_config from nupic.torch.modules import Flatten, KWinners2d, SparseWeights, SparseWeights2d config_file = "experiments.cfg" with open(config_file) as cf: config_init = parse_config(cf) exp = "sparseCNN2" # play with the config values here config = config_init[exp] config["name"] = exp config["use_dendrites"] = True config["use_batch_norm"] = False config["cnn_out_channels"] = (64, 64) config["cnn_percent_on"] = (0.12, 0.07) config["cnn_weight_sparsity"] = (0.15, 0.05) config["dendrites_per_cell"] = 2 config["batch_size"] = 64
def train(config, experiments, num_cpus, num_gpus, redis_address, show_list): print("config =", config.name) print("num_gpus =", num_gpus) print("num_cpus =", num_cpus) print("redis_address =", redis_address) # Use configuration file location as the project location. project_dir = os.path.dirname(config.name) project_dir = os.path.abspath(project_dir) print("project_dir =", project_dir) # Load and parse experiment configurations configs = parse_config(config, experiments, globals_param=globals()) if show_list: print("Experiments:", list(configs.keys())) return # Initialize ray cluster if redis_address is not None: ray.init(redis_address=redis_address, include_webui=True) num_cpus = 1 else: ray.init(num_cpus=num_cpus, num_gpus=num_gpus, local_mode=num_cpus == 1) # Run experiments resources_per_trial = {"cpu": 1, "gpu": num_gpus / num_cpus} print("resources_per_trial =", resources_per_trial) for exp in configs: print("experiment =", exp) config = configs[exp] config["name"] = exp # Stop criteria. Default to total number of iterations/epochs stop_criteria = {"training_iteration": config.get("iterations")} stop_criteria.update(config.get("stop", {})) print("stop_criteria =", stop_criteria) # Make sure local directories are relative to the project location path = config.get("path", None) if path and not os.path.isabs(path): config["path"] = os.path.join(project_dir, path) data_dir = config.get("data_dir", "data") if not os.path.isabs(data_dir): config["data_dir"] = os.path.join(project_dir, data_dir) tune.run( SpeechExperimentTune, name=config["name"], stop=stop_criteria, config=config, resources_per_trial=resources_per_trial, num_samples=config.get("repetitions", 1), local_dir=config.get("path", None), upload_dir=config.get("upload_dir", None), sync_function=config.get("sync_function", None), checkpoint_freq=config.get("checkpoint_freq", 0), checkpoint_at_end=config.get("checkpoint_at_end", False), export_formats=config.get("", None), search_alg=config.get("search_alg", None), scheduler=config.get("scheduler", None), verbose=config.get("verbose", 2), resume=config.get("resume", False), queue_trials=config.get("queue_trials", False), reuse_actors=config.get("reuse_actors", False), trial_executor=config.get("trial_executor", None), raise_on_failed_trial=config.get("raise_on_failed_trial", True), ) ray.shutdown()
def main(config, experiment, tablefmt, show_list): assert len(experiment) == 2, "Select 2 experiments (denseCNN2, sparseCNN2)" configs = parse_config(config, experiment, globals_param=globals()) if show_list: print("Experiments:", list(configs.keys())) return # Sort with dense configurations first configs = sorted(configs.items(), key=lambda x: 0 if x[0].lower().startswith("dense") else 1) params_table = [ [ "Network", "L1", "L2", "L3", "Output", "Total" ] ] l1_ratio = l2_ratio = l3_ratio = output_ratio = total_ratio = 1.0 for name, params in configs: input_shape = params["input_shape"] input_c, height, width = input_shape # CNN configuration cnn_out_channels = params["cnn_out_channels"] cnn_percent_on = params["cnn_percent_on"] cnn_weight_sparsity = params["cnn_weight_sparsity"] # Linear configuration linear_n = params["linear_n"] linear_percent_on = params["linear_percent_on"] weight_sparsity = params["weight_sparsity"] # Compute total non-zero weights in L1 l1_out_c = cnn_out_channels[0] l1_w = input_c * l1_out_c * KERNEL_SIZE * KERNEL_SIZE l1_w = l1_w * cnn_weight_sparsity[0] # Input density is 1, so L1 multiplies = output_shape * L1 weights l1_out_width = (width - KERNEL_SIZE + 1) l1_out_height = (height - KERNEL_SIZE + 1) l1_mul = l1_out_width * l1_out_height * l1_w # L1 Output after maxpool l1_out = [l1_out_c, l1_out_height / 2, l1_out_width / 2] # Compute total non-zero weights in L2 l2_out_c = cnn_out_channels[1] l2_w = l1_out_c * l2_out_c * KERNEL_SIZE * KERNEL_SIZE l2_w = l2_w * cnn_weight_sparsity[1] # L2 multiplies = input_sparsity * L2 weights * L2 output size l2_out_height = (l1_out[1] - KERNEL_SIZE + 1) l2_out_width = (l1_out[2] - KERNEL_SIZE + 1) l2_mul = cnn_percent_on[0] * l2_out_height * l2_out_width * l2_w # L2 Output after pool l2_out = [l2_out_c, l2_out_height / 2, l2_out_width / 2] # Compute total non-zero weights in L3 l3_w = np.prod(l2_out) * linear_n[0] l3_w = l3_w * weight_sparsity[0] # L3 multiplies = l2 sparsity * L3 weights l3_mul = cnn_percent_on[1] * l3_w # L3 Output l3_out = linear_n[0] l3_nnz_out = l3_out * linear_percent_on[0] # Output layer multiplies = l3 non-zero output * weights output_w = l3_out * params["num_classes"] output_mul = l3_nnz_out * output_w # Compute gain ratio against previous configuration l1_ratio = l1_mul / l1_ratio l2_ratio = l2_mul / l2_ratio l3_ratio = l3_mul / l3_ratio output_ratio = output_mul / output_ratio total_mul = l1_mul + l2_mul + l3_mul + output_mul total_ratio = total_mul / total_ratio params_table.append([name, "{:,.0f}".format(l1_mul), "{:,.0f}".format(l2_mul), "{:,.0f}".format(l3_mul), "{:,.0f}".format(output_mul), "{:,.0f}".format(total_mul), ]) params_table.append(["Computation Efficiency", "{:.0f} x".format(1.0 / l1_ratio), "{:.0f} x".format(1.0 / l2_ratio), "{:.0f} x".format(1.0 / l3_ratio), "{:.0f} x".format(1 / output_ratio), "{:.0f} x".format(1 / total_ratio), ]) print(tabulate(params_table, headers="firstrow", tablefmt=tablefmt, stralign="center", floatfmt=",.0f"))
def config_init(self, exp): with open(self.config_file) as cf: config = parse_config(cf) exp_config = config[exp] exp_config["name"] = exp return exp_config
def train(config, experiments, num_cpus, num_gpus, redis_address, show_list): print("config =", config.name) print("num_gpus =", num_gpus) print("num_cpus =", num_cpus) print("redis_address =", redis_address) # Use configuration file location as the project location. project_dir = os.path.dirname(config.name) project_dir = os.path.abspath(project_dir) print("project_dir =", project_dir) # Load and parse experiment configurations configs = parse_config(config, experiments, globals_param=globals()) if show_list: print("Experiments:", list(configs.keys())) return print("experiments =", list(configs.keys())) # Initialize ray cluster if redis_address is not None: ray.init(redis_address=redis_address, include_webui=True) else: ray.init(num_cpus=num_cpus, num_gpus=num_gpus, local_mode=num_cpus == 1) # Run experiments gpu_percent = 0 if num_gpus > 0: gpu_percent = configs.get("gpu_percentage", 0.5) resources_per_trial = {"cpu": 1, "gpu": gpu_percent} print("resources_per_trial =", resources_per_trial) for exp in configs: print("experiment =", exp) config = configs[exp] config["name"] = exp # Stop criteria. Default to total number of iterations/epochs stop_criteria = {"training_iteration": config.get("iterations")} stop_criteria.update(config.get("stop", {})) print("stop_criteria =", stop_criteria) # Make sure path and data_dir are relative to the project location, # handling both ~/nta and ../results style paths. path = config.get("path", ".") config["path"] = str(Path(path).expanduser().resolve()) data_dir = config.get("data_dir", "data") config["data_dir"] = str(Path(data_dir).expanduser().resolve()) tune.run( MNISTExperimentTune, name=config["name"], stop=stop_criteria, config=config, resources_per_trial=resources_per_trial, num_samples=config.get("repetitions", 1), local_dir=config.get("path", None), upload_dir=config.get("upload_dir", None), sync_function=config.get("sync_function", None), checkpoint_freq=config.get("checkpoint_freq", 0), checkpoint_at_end=config.get("checkpoint_at_end", False), export_formats=config.get("", None), search_alg=config.get("search_alg", None), scheduler=config.get("scheduler", None), verbose=config.get("verbose", 2), resume=config.get("resume", False), queue_trials=config.get("queue_trials", False), reuse_actors=config.get("reuse_actors", False), trial_executor=config.get("trial_executor", None), raise_on_failed_trial=config.get("raise_on_failed_trial", True), keep_checkpoints_num=1, checkpoint_score_attr="mean_accuracy", ) ray.shutdown()
def main(config, experiments, tablefmt): # Use configuration file location as the project location. project_dir = os.path.dirname(config.name) project_dir = os.path.abspath(project_dir) print("project_dir =", project_dir) test_scores_table = [["Network", "Test Score", "Noise Score"]] # Load and parse experiment configurations configs = parse_config(config, experiments, globals_param=globals()) # Select tags ignoring seed def key_func(x): re.split("[,_]", re.sub(",|\\d+_|seed=\\d+", "", x["experiment_tag"])) for exp in configs: config = configs[exp] # Load experiment data experiment_path = os.path.join(project_dir, config["path"], exp) experiment_state = load_ray_tune_experiment( experiment_path=experiment_path, load_results=True) # Go through all checkpoints in the experiment all_checkpoints = experiment_state["checkpoints"] # Group checkpoints by tags checkpoint_groups = { k[0]: list(v) for k, v in groupby(sorted(all_checkpoints, key=key_func), key=key_func) } for tag in checkpoint_groups: checkpoints = checkpoint_groups[tag] num_exps = len(checkpoints) test_scores = np.zeros(num_exps) noise_scores = np.zeros(num_exps) for i, checkpoint in enumerate(checkpoints): results = checkpoint["results"] if results is None: continue # For each checkpoint select the epoch with the best accuracy as # the best epoch best_result = max(results, key=lambda x: x["mean_accuracy"]) test_scores[i] = best_result["mean_accuracy"] * 100.0 # Load noise score logdir = os.path.join(experiment_path, os.path.basename(checkpoint["logdir"])) filename = os.path.join(logdir, "noise.json") with open(filename, "r") as f: noise = json.load(f) noise_scores[i] = sum(x["total_correct"] for x in list(noise.values())) test_score = "{0:.2f} ± {1:.2f}".format(test_scores.mean(), test_scores.std()) noise_score = "{0:,.0f} ± {1:.2f}".format(noise_scores.mean(), noise_scores.std()) test_scores_table.append( ["{} {}".format(exp, tag), test_score, noise_score]) print() print(tabulate(test_scores_table, headers="firstrow", tablefmt=tablefmt))
def run(config, experiments, num_cpus, num_gpus, redis_address, noise_values): print("config =", config.name) print("experiments =", experiments) print("num_gpus =", num_gpus) print("num_cpus =", num_cpus) print("redis_address =", redis_address) print("noise_values =", noise_values) # Use configuration file location as the project location. project_dir = os.path.dirname(config.name) project_dir = os.path.abspath(project_dir) print("project_dir =", project_dir) # Download dataset data_dir = os.path.join(project_dir, "data") datasets.CIFAR10(data_dir, download=True, train=True) # Initialize ray cluster if redis_address is not None: ray.init(redis_address=redis_address, include_webui=True) else: ray.init(num_cpus=num_cpus, num_gpus=num_gpus, local_mode=num_cpus == 1) # Load and parse experiment configurations configs = parse_config(config, experiments, globals(), locals()) # Run all experiments in parallel ray_trials = [] for exp in configs: config = configs[exp] # noise experiment tune configuration noise_config = { "iterations": 1, "noise": { "grid_search": list(noise_values) } } # Download results from S3 when running on the cluster if redis_address is not None and "upload_dir" in config: upload_dir = config["upload_dir"] download_s3_results("{}/{}".format(upload_dir, exp), os.path.join(config["path"], exp)) # Store noise results with original results in S3 noise_config["upload_dir"] = "{}".format(upload_dir) noise_config[ "sync_function"] = "aws s3 sync `dirname {local_dir}` {remote_dir}/`basename $(dirname {local_dir})`" # noqa E501 else: noise_config.pop("upload_dir", None) noise_config.pop("sync_function", None) # Load experiment results experiment_path = os.path.join(project_dir, config["path"], exp) experiment_state = load_ray_tune_experiment(experiment_path, load_results=True) all_experiments = experiment_state["checkpoints"] for experiment in all_experiments: # Make logs relative to experiment path logdir = experiment["logdir"] logpath = os.path.join(experiment_path, os.path.basename(logdir)) # Check for experiment results results = experiment["results"] if results is None: continue # Get best scoring model checkpoint from results best_result = max(results, key=lambda x: x["mean_accuracy"]) epoch = best_result["training_iteration"] checkpoint_path = os.path.join(logpath, "checkpoint_{}".format(epoch)) if os.path.exists(checkpoint_path): # Update data path model_config = best_result["config"] model_config["data_dir"] = data_dir # Run noise tests noise_config.update({ "name": experiment["experiment_tag"], "path": os.path.join(project_dir, "results", "noise", exp), "checkpoint_path": checkpoint_path, "model_config": model_config, }) ray_trials.append( run_experiment.remote( noise_config, MobileNetNoiseTune, num_cpus=1, num_gpus=min(1, num_gpus), )) # Wait for all experiments to complete ray.get(ray_trials) ray.shutdown()
def train(config, experiments, num_cpus, num_gpus, redis_address, show_list): print("config =", config.name) print("num_gpus =", num_gpus) print("num_cpus =", num_cpus) print("redis_address =", redis_address) # Use configuration file location as the project location. project_dir = os.path.dirname(config.name) project_dir = os.path.abspath(project_dir) print("project_dir =", project_dir) # Load and parse experiment configurations configs = parse_config(config, experiments, globals=globals()) if show_list: print("Experiments:", list(configs.keys())) return # Initialize ray cluster if redis_address is not None: ray.init(redis_address=redis_address, include_webui=True) num_cpus = 1 else: ray.init(num_cpus=num_cpus, num_gpus=num_gpus, local_mode=num_cpus == 1) # Run experiments resources_per_trial = {"cpu": 1, "gpu": num_gpus / num_cpus} print("resources_per_trial =", resources_per_trial) for exp in configs: print("experiment =", exp) config = configs[exp] config["name"] = exp # Stop criteria. Default to total number of iterations/epochs stop_criteria = {"training_iteration": config.get("iterations")} stop_criteria.update(config.get("stop", {})) print("stop_criteria =", stop_criteria) # Make sure local directories are relative to the project location path = config.get("path", None) if path and not os.path.isabs(path): config["path"] = os.path.join(project_dir, path) data_dir = config.get("data_dir", "data") if not os.path.isabs(data_dir): config["data_dir"] = os.path.join(project_dir, data_dir) tune.run( SpeechExperimentTune, name=config["name"], stop=stop_criteria, config=config, resources_per_trial=resources_per_trial, num_samples=config.get("repetitions", 1), local_dir=config.get("path", None), upload_dir=config.get("upload_dir", None), sync_function=config.get("sync_function", None), checkpoint_freq=config.get("checkpoint_freq", 0), checkpoint_at_end=config.get("checkpoint_at_end", False), export_formats=config.get("", None), search_alg=config.get("search_alg", None), scheduler=config.get("scheduler", None), verbose=config.get("verbose", 2), resume=config.get("resume", False), queue_trials=config.get("queue_trials", False), reuse_actors=config.get("reuse_actors", False), trial_executor=config.get("trial_executor", None), raise_on_failed_trial=config.get("raise_on_failed_trial", True) ) ray.shutdown()
def main(config, experiments, tablefmt): # The table we use in the paper test_scores_table = [["Network", "Test Score", "Noise Score", "Params"]] # A more detailed table test_scores_table_long = [[ "Network", "Test Score", "Noise Score", "Noise Accuracy", "Total Entropy", "Nonzero Parameters", "Num Trials", "Session" ]] # Load and parse experiment configurations configs = parse_config(config, experiments, globals_param=globals()) # Use the appropriate plus/minus sign for latex if tablefmt == "grid": pm = "±" else: pm = "$\\pm$" # Select tags ignoring seed value def key_func(x): s = re.split("[,]", re.sub(",|\\d+_|seed=\\d+", "", x["experiment_tag"])) if len(s[0]) == 0: return [" "] return s for exp in configs: config = configs[exp] # Make sure path and data_dir are relative to the project location, # handling both ~/nta and ../results style paths. path = config.get("path", ".") config["path"] = str(Path(path).expanduser().resolve()) data_dir = config.get("data_dir", "data") config["data_dir"] = str(Path(data_dir).expanduser().resolve()) # Load experiment data experiment_path = os.path.join(config["path"], exp) try: states = load_ray_tune_experiments(experiment_path=experiment_path, load_results=True) except RuntimeError: # print("Could not locate experiment state for " + exp + " ...skipping") continue for experiment_state in states: # Go through all checkpoints in the experiment all_checkpoints = experiment_state["checkpoints"] # Group checkpoints by tags checkpoint_groups = { k[0]: list(v) for k, v in groupby(sorted(all_checkpoints, key=key_func), key=key_func) } for tag in checkpoint_groups: checkpoints = checkpoint_groups[tag] num_exps = len(checkpoints) test_scores = np.zeros(num_exps) noise_scores = np.zeros(num_exps) noise_accuracies = np.zeros(num_exps) noise_samples = np.zeros(num_exps) nonzero_params = np.zeros(num_exps) entropies = np.zeros(num_exps) try: for i, checkpoint in enumerate(checkpoints): results = checkpoint["results"] if results is None: continue # For each checkpoint select the epoch with the best accuracy as # the best epoch best_result = max(results, key=lambda x: x["mean_accuracy"]) test_scores[i] = best_result["mean_accuracy"] entropies[i] = best_result["entropy"] # print("best result:", best_result) # Load noise score logdir = os.path.join( experiment_path, os.path.basename(checkpoint["logdir"])) filename = os.path.join(logdir, "noise.json") if os.path.exists(filename): with open(filename, "r") as f: noise = json.load(f) noise_scores[i] = sum( x["total_correct"] for x in list(noise.values())) noise_samples[i] = sum( x["total_samples"] for x in list(noise.values())) else: print("No noise file for " + experiment_path + " ...skipping") continue noise_accuracies[i] = (float(100.0 * noise_scores[i]) / noise_samples[i]) nonzero_params[i] = max(x["non_zero_parameters"] for x in list(noise.values())) except Exception: print("Problem with checkpoint group" + tag + " in " + exp + " ...skipping") continue test_score = "{0:.2f} {1:} {2:.2f}".format( test_scores.mean(), pm, test_scores.std()) entropy = "{0:.2f} {1:} {2:.2f}".format( entropies.mean(), pm, entropies.std()) noise_score = "{0:,.0f} {1:} {2:.2f}".format( noise_scores.mean(), pm, noise_scores.std()) noise_accuracy = "{0:,.2f} {1:} {2:.2f}".format( noise_accuracies.mean(), pm, noise_accuracies.std()) nonzero = "{0:,.0f}".format(nonzero_params.mean()) test_scores_table.append([ "{} {}".format(exp, tag), test_score, noise_accuracy, nonzero ]) test_scores_table_long.append([ "{} {}".format(exp, tag), test_score, noise_score, noise_accuracy, entropy, nonzero, num_exps, experiment_state["runner_data"]["_session_str"] ]) print() print(tabulate(test_scores_table, headers="firstrow", tablefmt=tablefmt)) print() print( tabulate(test_scores_table_long, headers="firstrow", tablefmt=tablefmt))
def run(config, experiments, num_cpus, num_gpus, redis_address, noise_values): print("config =", config.name) print("experiments =", experiments) print("num_gpus =", num_gpus) print("num_cpus =", num_cpus) print("redis_address =", redis_address) print("noise_values =", noise_values) # Use configuration file location as the project location. projectDir = os.path.dirname(config.name) projectDir = os.path.abspath(projectDir) print("projectDir =", projectDir) # Download dataset data_dir = os.path.join(projectDir, "data") datasets.CIFAR10(data_dir, download=True, train=True) # Initialize ray cluster if redis_address is not None: ray.init(redis_address=redis_address, include_webui=True) else: ray.init(num_cpus=num_cpus, num_gpus=num_gpus, local_mode=num_cpus == 1) # Load and parse experiment configurations configs = parse_config(config, experiments, globals(), locals()) # Run all experiments in parallel ray_trials = [] for exp in configs: config = configs[exp] # noise experiment tune configuration noise_config = { "iterations": 1, "noise": {"grid_search": list(noise_values)} } # Download results from S3 when running on the cluster if redis_address is not None and "upload_dir" in config: upload_dir = config["upload_dir"] download_s3_results("{}/{}".format(upload_dir, exp), os.path.join(config["path"], exp)) # Store noise results with original results in S3 noise_config["upload_dir"] = "{}".format(upload_dir) noise_config["sync_function"] = "aws s3 sync `dirname {local_dir}` {remote_dir}/`basename $(dirname {local_dir})`" else: noise_config.pop("upload_dir", None) noise_config.pop("sync_function", None) # Load experiment results experiment_path = os.path.join(projectDir, config["path"], exp) experiment_state = load_ray_tune_experiment(experiment_path, load_results=True) all_experiments = experiment_state["checkpoints"] for experiment in all_experiments: # Make logs relative to experiment path logdir = experiment["logdir"] logpath = os.path.join(experiment_path, os.path.basename(logdir)) # Check for experiment results results = experiment["results"] if results is None: continue # Get best scoring model checkpoint from results best_result = max(results, key=lambda x: x["mean_accuracy"]) epoch = best_result["training_iteration"] checkpoint_path = os.path.join(logpath, "checkpoint_{}".format(epoch)) if os.path.exists(checkpoint_path): # Update data path model_config = best_result["config"] model_config["data_dir"] = data_dir # Run noise tests noise_config.update({ "name": experiment["experiment_tag"], "path": os.path.join(projectDir, "results", "noise", exp), "checkpoint_path": checkpoint_path, "model_config": model_config }) ray_trials.append( run_experiment.remote(noise_config, MobileNetNoiseTune, num_cpus=1, num_gpus=min(1, num_gpus))) # Wait for all experiments to complete ray.get(ray_trials) ray.shutdown()
def main(config, experiment, tablefmt, show_list): configs = parse_config(config, experiment, globals_param=globals()) if show_list: print("Experiments:", list(configs.keys())) return params_table = [[ "Network", "L1 Filters", "L1 Act Sparsity", "L1 Wt Sparsity", "L2 Filters", "L2 Act Sparsity", "L2 Wt Sparsity", "L3 N", "L3 Act Sparsity", "Wt Sparsity", ]] params_table1 = [[ "Network", "L1 Channels", "L2 Channels", "L3 N", ]] params_table2 = [[ "Network", "L1 Activation Sparsity", "L2 Activation Sparsity", "L3 Activation Sparsity", ]] params_table3 = [[ "Network", "L1 Weight Sparsity", "L2 Weight Sparsity", "L3 Weight Sparsity", ]] for name, params in configs.items(): linear_n = params["linear_n"] linear_percent_on = params["linear_percent_on"] weight_sparsity = params["weight_sparsity"] cnn_percent_on = params["cnn_percent_on"] cnn_out_channels = params["cnn_out_channels"] cnn_weight_sparsity = params["cnn_weight_sparsity"] l3_n = linear_n[0] if linear_percent_on[0] > 0.50: l3_sp = "ReLU" else: l3_sp = "{0:.1f}%".format(100 * (1.0 - linear_percent_on[0])) wt_sp = "{0}%".format(100 * (1.0 - weight_sparsity[0])) l1_percent_on = cnn_percent_on[0] l1_wt_sparsity = cnn_weight_sparsity[0] if len(cnn_percent_on) == 2: l2_percent_on = cnn_percent_on[1] l2_wt_sparsity = cnn_weight_sparsity[1] else: l2_percent_on = None l2_wt_sparsity = None l1_f = cnn_out_channels[0] if l1_percent_on > 0.50: l1_sp = "ReLU" else: l1_sp = "{0:.1f}%".format(100 * (1.0 - l1_percent_on)) l1_wt = "{0:.1f}%".format(100 * (1.0 - l1_wt_sparsity)) if len(cnn_out_channels) == 2: l2_f = cnn_out_channels[1] if l2_percent_on > 0.50: l2_sp = "ReLU" else: l2_sp = "{0:.1f}%".format(100 * (1.0 - l2_percent_on)) l2_wt = "{0:.1f}%".format(100 * (1.0 - l2_wt_sparsity)) else: l2_f = None l2_sp = None l2_wt = None params_table.append( [name, l1_f, l1_sp, l1_wt, l2_f, l2_sp, l2_wt, l3_n, l3_sp, wt_sp]) params_table1.append([name, l1_f, l2_f, l3_n]) params_table2.append([name, l1_sp, l2_sp, l3_sp]) params_table3.append([name, l1_wt, l2_wt, wt_sp]) print() print( tabulate( params_table, headers="firstrow", tablefmt=tablefmt, stralign="center", )) print() print( tabulate(params_table1, headers="firstrow", tablefmt=tablefmt, stralign="left", numalign="center")) print() print( tabulate(params_table2, headers="firstrow", tablefmt=tablefmt, stralign="left", numalign="center")) print() print( tabulate(params_table3, headers="firstrow", tablefmt=tablefmt, stralign="left", numalign="center"))