def main(*args, **kwargs): nasbench = nasbench_api.NASBench(FLAGS.path_to_nasbench) module = nasbench.fixed_statistics[FLAGS.hash_key] spec = model_spec.ModelSpec(module['module_adjacency'], module['module_operations']) config = nasbench_config.build_config() for flag in FLAGS.flags_by_module_dict()[args[0][0]]: config[flag.name] = flag.value config['use_tpu'] = False config['use_KD'] = False config['intermediate_evaluations'] = ['1.0'] trainset_multipier = FLAGS.trainset_part_percentage / 100.0 config['num_train'] = int(config['num_train'] * trainset_multipier) config['num_train_eval'] = int(config['num_train_eval'] * trainset_multipier) config['num_augment'] = int(config['num_augment'] * trainset_multipier) logging.info("Prepare KD dataset") dataset_files = FLAGS.train_data_files + [ FLAGS.valid_data_file, FLAGS.test_data_file, FLAGS.sample_data_file ] prepare_kd_dataset(spec, config, FLAGS.save_path, dataset_files, FLAGS.new_dataset_path, FLAGS.trainset_part_percentage)
def main(): config = _config.build_config() config['use_tpu'] = False config['train_epochs'] = 1 matrix=[[0, 1, 1, 1, 0, 1, 0], # input layer [0, 0, 0, 0, 0, 0, 1], # 1x1 conv [0, 0, 0, 0, 0, 0, 1], # 3x3 conv [0, 0, 0, 0, 1, 0, 0], # 5x5 conv (replaced by two 3x3's) [0, 0, 0, 0, 0, 0, 1], # 5x5 conv (replaced by two 3x3's) [0, 0, 0, 0, 0, 0, 1], # 3x3 max-pool [0, 0, 0, 0, 0, 0, 0]] # output layer # Operations at the vertices of the module, matches order of matrix labels=['input', 'conv1x1-bn-relu', 'conv3x3-bn-relu', 'conv3x3-bn-relu', 'conv3x3-bn-relu', 'maxpool3x3', 'output'] model_dir = '../my_model_dir' print(tf.train.latest_checkpoint(model_dir)) sess = tf.Session() saver = tf.train.import_meta_graph(tf.train.latest_checkpoint(model_dir)+'.meta') saver.restore(sess, tf.train.latest_checkpoint(model_dir)) print('OK')
def main(): config = _config.build_config() config['use_tpu'] = False config['train_data_files'] = ['train_1.tfrecords', 'train_2.tfrecords', 'train_3.tfrecords', 'train_4.tfrecords'] config['valid_data_file'] = ['validation.tfrecords'] config['test_data_file'] = ['test.tfrecords'] config['sample_data_file'] = ['sample.tfrecords'] config['train_epochs'] = 1 config['use_KD'] = False matrix=[[0, 1, 1, 1, 0, 1, 0], # input layer [0, 0, 0, 0, 0, 0, 1], # 1x1 conv [0, 0, 0, 0, 0, 0, 1], # 3x3 conv [0, 0, 0, 0, 1, 0, 0], # 5x5 conv (replaced by two 3x3's) [0, 0, 0, 0, 0, 0, 1], # 5x5 conv (replaced by two 3x3's) [0, 0, 0, 0, 0, 0, 1], # 3x3 max-pool [0, 0, 0, 0, 0, 0, 0]] # output layer # Operations at the vertices of the module, matches order of matrix labels=['input', 'conv1x1-bn-relu', 'conv3x3-bn-relu', 'conv3x3-bn-relu', 'conv3x3-bn-relu', 'maxpool3x3', 'output'] matrix = np.array(matrix) labels = np.array(labels) spec = model_spec.ModelSpec(matrix, labels) model_dir = '../data/tmp' meta = evaluate.train_and_evaluate(spec, config, model_dir) output_file = os.path.join(model_dir, RESULTS_FILE) with tf.gfile.Open(output_file, 'w') as f: json.dump(meta, f, cls=NumpyEncoder) print('OK') print(spec.__dict__)
def network_fn(args): genotype = arch_to_genotype(args.arch, args.policy) if args.policy.lower() == 'nasbench': model = NasBenchNet(3, genotype, _config.build_config()) elif args.policy.lower() in ['nao', 'darts', 'enas', 'random']: CIFAR_CLASSES = 10 model = NetworkCIFAR(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) else: raise NotImplementedError return model, genotype
def __init__(self, models_file, output_dir, worker_id=0, total_workers=1, model_id_regex='^', sort_keys=False): self.config = _config.build_config() with tf.io.gfile.GFile(models_file) as f: self.models = json.load(f) self.remainders = None self.ordered_keys = None if FLAGS.remainders_file: # Run only the modules and repeat numbers specified with tf.io.gfile.GFile(FLAGS.remainders_file) as f: self.remainders = json.load(f) self.remainders = sorted(self.remainders) self.num_models = len(self.remainders) self.total_work_units = self.num_models else: # Filter keys to only those that fit the regex and order them so all # workers see a canonical ordering. regex = re.compile(model_id_regex) evaluated_keys = [ key for key in self.models.keys() if regex.match(key) ] if sort_keys: self.ordered_keys = sorted(evaluated_keys) else: self.ordered_keys = evaluated_keys self.num_models = len(self.ordered_keys) self.total_work_units = self.num_models * self.config['num_repeats'] self.total_workers = total_workers # If the worker is recovering from a restart, figure out where to restart worker_recovery_dir = os.path.join(output_dir, '_recovery') tf.io.gfile.makedirs( worker_recovery_dir) # Silently succeeds if exists self.recovery_file = os.path.join(worker_recovery_dir, str(worker_id)) if tf.io.gfile.exists(self.recovery_file): with tf.io.gfile.GFile(self.recovery_file) as f: self.current_index = int(f.read()) else: self.current_index = worker_id with tf.io.gfile.GFile(self.recovery_file, 'w') as f: f.write(str(self.current_index)) assert self.current_index % self.total_workers == worker_id self.output_dir = output_dir
def main(_): config = _config.build_config() # The default settings in config are exactly what was used to generate the # dataset of models. However, given more epochs and a different learning rate # schedule, it is possible to get higher accuracy. config['train_epochs'] = 200 config['lr_decay_method'] = 'STEPWISE' config['train_seconds'] = -1 # Disable training time limit spec = create_best_nasbench_spec(config) data = evaluate.augment_and_evaluate(spec, config, FLAGS.model_dir) tf.compat.v1.logging.info(data)
def __init__(self, dataset_file): """Initialize dataset, this should only be done once.""" # TODO(chrisying): download the file directly if dataset_file not provided self.config = config.build_config() print('Loading dataset from file... This may take a few minutes...') start = time.time() self.dataset = {} for serialized_row in tf.python_io.tf_record_iterator(dataset_file): data_point = {} row = json.loads(serialized_row) module_hash = row[0] metrics = model_metrics_pb2.ModelMetrics.FromString( row[3].decode('base64')) # TODO(chrisying): is it useful to keep the original adjacency and # operations actually evaluated? Leaving this commented out because it # saves a lot of memory. ''' module_adjacency = row[1] module_operations = row[2] adjacency = np.array(map(int, list(metrics['module_adjacency']))) dim = int(math.sqrt(len(adjacency))) assert dim * dim == len(adjacency) data_point['module_adjacency'] = adjacency.reshape((dim, dim)) data_point['module_operations'] = metrics['module_operations'].split(',') ''' data_point['trainable_parameters'] = metrics.trainable_parameters final_evaluation = metrics.evaluation_data[-1] data_point['training_time'] = final_evaluation.training_time data_point['train_accuracy'] = final_evaluation.train_accuracy data_point[ 'validation_accuracy'] = final_evaluation.validation_accuracy data_point['test_accuracy'] = final_evaluation.test_accuracy # TODO(chrisying): when we support multiple epoch lengths, dict key will # include the epoch count as well if module_hash not in self.dataset: self.dataset[module_hash] = [] self.dataset[module_hash].append(data_point) elapsed = time.time() - start print('Loaded dataset in %d seconds' % elapsed) self.history = {} self.total_time_spent = 0 self.total_epochs_spent = 0
def main(*args, **kwargs): nasbench = nasbench_api.NASBench(FLAGS.path_to_nasbench) module = nasbench.fixed_statistics[FLAGS.hash_key] spec = model_spec.ModelSpec(module['module_adjacency'], module['module_operations']) config = nasbench_config.build_config() for flag in FLAGS.flags_by_module_dict()[args[0][0]]: config[flag.name] = flag.value config['use_tpu'] = False config['use_KD'] = False config['intermediate_evaluations'] = ['1.0'] trainset_multipier = FLAGS.trainset_part_percentage / 100.0 config['num_train'] = int(config['num_train'] * trainset_multipier) config['num_train_eval'] = int(config['num_train_eval'] * trainset_multipier) config['num_augment'] = int(config['num_augment'] * trainset_multipier) logging.info("Train and evaluate with config\n{}\n and spec\n{}".format(config, spec)) train(spec, config, FLAGS.save_path)
def __init__(self, dataset_file, seed=None): """Initialize dataset, this should only be done once per experiment. Args: dataset_file: path to .tfrecord file containing the dataset. seed: random seed used for sampling queried models. Two NASBench objects created with the same seed will return the same data points when queried with the same models in the same order. By default, the seed is randomly generated. """ self.config = config.build_config() random.seed(seed) print('Loading dataset from file... This may take a few minutes...') start = time.time() # Stores the fixed statistics that are independent of evaluation (i.e., # adjacency matrix, operations, and number of parameters). # hash --> metric name --> scalar self.fixed_statistics = {} # Stores the statistics that are computed via training and evaluating the # model on CIFAR-10. Statistics are computed for multiple repeats of each # model at each max epoch length. # hash --> epochs --> repeat index --> metric name --> scalar self.computed_statistics = {} # Valid queriable epoch lengths. {4, 12, 36, 108} for the full dataset or # {108} for the smaller dataset with only the 108 epochs. self.valid_epochs = set() for serialized_row in tf.python_io.tf_record_iterator(dataset_file): # Parse the data from the data file. module_hash, epochs, raw_adjacency, raw_operations, raw_metrics = ( json.loads(serialized_row.decode('utf-8'))) dim = int(np.sqrt(len(raw_adjacency))) adjacency = np.array([int(e) for e in list(raw_adjacency)], dtype=np.int8) adjacency = np.reshape(adjacency, (dim, dim)) operations = raw_operations.split(',') metrics = model_metrics_pb2.ModelMetrics.FromString( base64.b64decode(raw_metrics)) if module_hash not in self.fixed_statistics: # First time seeing this module, initialize fixed statistics. new_entry = {} new_entry['module_adjacency'] = adjacency new_entry['module_operations'] = operations new_entry[ 'trainable_parameters'] = metrics.trainable_parameters self.fixed_statistics[module_hash] = new_entry self.computed_statistics[module_hash] = {} self.valid_epochs.add(epochs) if epochs not in self.computed_statistics[module_hash]: self.computed_statistics[module_hash][epochs] = [] # Each data_point consists of the metrics recorded from a single # train-and-evaluation of a model at a specific epoch length. data_point = {} # Note: metrics.evaluation_data[0] contains the computed metrics at the # start of training (step 0) but this is unused by this API. # Evaluation statistics at the half-way point of training half_evaluation = metrics.evaluation_data[1] data_point['halfway_training_time'] = half_evaluation.training_time data_point[ 'halfway_train_accuracy'] = half_evaluation.train_accuracy data_point['halfway_validation_accuracy'] = ( half_evaluation.validation_accuracy) data_point['halfway_test_accuracy'] = half_evaluation.test_accuracy # Evaluation statistics at the end of training final_evaluation = metrics.evaluation_data[2] data_point['final_training_time'] = final_evaluation.training_time data_point[ 'final_train_accuracy'] = final_evaluation.train_accuracy data_point['final_validation_accuracy'] = ( final_evaluation.validation_accuracy) data_point['final_test_accuracy'] = final_evaluation.test_accuracy self.computed_statistics[module_hash][epochs].append(data_point) elapsed = time.time() - start print('Loaded dataset in %d seconds' % elapsed) self.history = {} self.training_time_spent = 0.0 self.total_epochs_spent = 0
import networkx as nx import torch import torch.nn as nn import numpy as np import IPython from torch.distributions import Categorical from search_policies.cnn.search_space.nas_bench.model_search import NasBenchCellSearch from search_policies.cnn.search_space.nas_bench.nasbench_api_v2 import ModelSpec_v2 from search_policies.cnn.search_space.nas_bench.model_builder import compute_vertex_channels from nasbench.lib import config as _config from search_policies.cnn.search_space.nas_bench.operations import conv_bn_relu, MixedVertexDARTS from search_policies.cnn.search_space.nas_bench.model import NasBenchNet, NasBenchCell import torch.nn.functional as F NASBENCH_CONFIG = _config.build_config() class ModelSpec_DARTS(ModelSpec_v2): # This is a dynamic model spec for NasBenchNetSearchDarts. # n = intermediate nodes. alpha_topology = None AVAILABLE_OPS = ['conv3x3-bn-relu', 'conv1x1-bn-relu', 'maxpool3x3'] # This is size [n + 1, n], [:-1, :] is as original matrix without input and output [1:-1, 1:-1]. # [-1, :] is used for zero operation # to extract operation, use softmax([:, node_id]). # m = num_ops alpha_ops = None # size [m, n]. sample m operations accordingly. # Extract operation: softmax([:, node_id]).
def __init__(self, dataset_file, seed=None): """Initialize dataset, this should only be done once per experiment. Args: dataset_file: path to .tfrecord file containing the dataset. seed: random seed used for sampling queried models. Two NASBench objects created with the same seed will return the same data points when queried with the same models in the same order. By default, the seed is randomly generated. """ # TODO(chrisying): download the file directly if dataset_file not provided self.config = config.build_config() random.seed(seed) print('Loading dataset from file... This may take a few minutes...') start = time.time() self.dataset = {} self.valid_epochs = set() for serialized_row in tf.python_io.tf_record_iterator(dataset_file): data_point = {} row = json.loads(serialized_row) module_hash = row[0] max_epochs = row[1] self.valid_epochs.add(max_epochs) metrics = model_metrics_pb2.ModelMetrics.FromString( row[4].decode('base64')) # TODO(chrisying): is it useful to keep the original adjacency and # operations actually evaluated? Leaving this commented out because it # saves a lot of memory. ''' module_adjacency = row[2] module_operations = row[3] adjacency = np.array(map(int, list(metrics['module_adjacency']))) dim = int(math.sqrt(len(adjacency))) assert dim * dim == len(adjacency) data_point['module_adjacency'] = adjacency.reshape((dim, dim)) data_point['module_operations'] = metrics['module_operations'].split(',') ''' data_point['trainable_parameters'] = metrics.trainable_parameters half_evaluation = metrics.evaluation_data[1] final_evaluation = metrics.evaluation_data[2] data_point['training_time'] = (half_evaluation.training_time, final_evaluation.training_time) data_point['train_accuracy'] = (half_evaluation.train_accuracy, final_evaluation.train_accuracy) data_point['validation_accuracy'] = (half_evaluation.validation_accuracy, final_evaluation.validation_accuracy) data_point['test_accuracy'] = (half_evaluation.test_accuracy, final_evaluation.test_accuracy) key = (module_hash, max_epochs) if key not in self.dataset: self.dataset[key] = [] self.dataset[key].append(data_point) elapsed = time.time() - start print('Loaded dataset in %d seconds' % elapsed) self.history = {} self.training_time_spent = 0 self.total_epochs_spent = 0
import numpy as np import matplotlib matplotlib.use('Agg') from matplotlib import pyplot as plt import pandas as pd import seaborn as sns import tensorflow as tf from nasbench import api from nasbench.lib import model_spec, model_builder, config, cifar nb = api.NASBench('nasbench_only108.tfrecord') data = {'params': [], 'time': [], 'acc': [], 'flops': []} cfg = config.build_config() cfg['train_data_files'] = [ 'nasbench/scripts/train_1.tfrecords', 'nasbench/scripts/train_2.tfrecords', 'nasbench/scripts/train_3.tfrecords', 'nasbench/scripts/train_4.tfrecords' ] cfg['use_tpu'] = False keys = list() count = 0 for key in nb.hash_iterator(): keys.append(key) count += 1 def keytotuple(key): cur_network_data = nb.get_metrics_from_hash(key)
help='train and test best model at this budget point') parser.add_argument('--verbose', type=int, default=1) args = parser.parse_args() epochs = args.epochs verbose = args.verbose val_split = 0.2 num_classes = 10 best = np.load('best_file.npy', allow_pickle=True).item() (x_train, y_train), (x_test, y_test) = cifar10.load_data() y_train = to_categorical(y_train, num_classes) y_test = to_categorical(y_test, num_classes) best_enc = best[epochs][0] model_config = config.build_config() model_spec = ModelSpec(matrix=best_enc['module_adjacency'], ops=best_enc['module_operations'], data_format=model_config['data_format']) batch_size = 250 # model_config['num_stacks'] = 0 print(model_config) inputs = tf.keras.layers.Input(x_train.shape[1:], batch_size) net_outputs = build_keras_model(model_spec, inputs, model_spec.ops, model_config) model = tf.keras.Model(inputs=inputs, outputs=net_outputs) num_train_imgs = int(x_train.shape[0]*(1-val_split))
from nasbench import api from nasbench.lib import graph_util, config from aw_nas import utils from aw_nas.utils.exception import expect from aw_nas.common import SearchSpace from aw_nas.rollout.base import BaseRollout from aw_nas.controller.base import BaseController from aw_nas.evaluator.base import BaseEvaluator from aw_nas.rollout.compare import CompareRollout from aw_nas.evaluator.arch_network import ArchEmbedder from aw_nas.utils import DenseGraphConvolution, DenseGraphFlow VERTICES = 7 MAX_EDGES = 9 _nasbench_cfg = config.build_config() def _literal_np_array(arr): if arr is None: return None return "np.array({})".format(np.array2string(arr, separator=",").replace("\n", " ")) class _ModelSpec(api.ModelSpec): def __repr__(self): return "_ModelSpec({}, {}; pruned_matrix={}, pruned_ops={})".format( _literal_np_array(self.original_matrix), self.original_ops, _literal_np_array(self.matrix), self.ops) def hash_spec(self, *args, **kwargs): return super(_ModelSpec, self).hash_spec(_nasbench_cfg["available_ops"])
if config['data_format'] == 'channels_last': net = tf.reduce_mean(net, [1, 2]) elif config['data_format'] == 'channels_first': net = tf.reduce_mean(net, [2, 3]) else: raise ValueError('invalid data_format') # Fully-connected layer to labels logits = tf.layers.dense(inputs=net, units=config['num_labels']) return logits if __name__ == '__main__': best = np.load('best_file.npy', allow_pickle=True).item() (x_train, y_train), (x_test, y_test) = cifar10.load_data() best_36 = best[36][0] model_spec = api.ModelSpec(matrix=best_36['module_adjacency'], ops=best_36['module_operations']) print(x_train.shape) print(config.build_config()) print(best_36['module_adjacency'], best_36['module_operations']) inputs = tf.placeholder(tf.float32, shape=x_train.shape, name='Input') # inputs = InputLayer(input_shape=x_train.shape[1:]) # inputs = tf.data.Dataset.from_tensor_slices((x_train, y_train)) model = build_model(inputs, model_spec, config.build_config()) # with tf.Session() as sess: # writer = tf.summary.FileWriter("model-output", sess.graph) # print(sess.run(model)) # writer.close()