def main(*args, **kwargs):
nasbench = nasbench_api.NASBench(FLAGS.path_to_nasbench)
module = nasbench.fixed_statistics[FLAGS.hash_key]
spec = model_spec.ModelSpec(module['module_adjacency'],
module['module_operations'])
config = nasbench_config.build_config()
for flag in FLAGS.flags_by_module_dict()[args[0][0]]:
config[flag.name] = flag.value
config['use_tpu'] = False
config['use_KD'] = False
config['intermediate_evaluations'] = ['1.0']
trainset_multipier = FLAGS.trainset_part_percentage / 100.0
config['num_train'] = int(config['num_train'] * trainset_multipier)
config['num_train_eval'] = int(config['num_train_eval'] *
trainset_multipier)
config['num_augment'] = int(config['num_augment'] * trainset_multipier)
logging.info("Prepare KD dataset")
dataset_files = FLAGS.train_data_files + [
FLAGS.valid_data_file, FLAGS.test_data_file, FLAGS.sample_data_file
]
prepare_kd_dataset(spec, config, FLAGS.save_path, dataset_files,
FLAGS.new_dataset_path, FLAGS.trainset_part_percentage)
def main():
config = _config.build_config()
config['use_tpu'] = False
config['train_epochs'] = 1
matrix=[[0, 1, 1, 1, 0, 1, 0], # input layer
[0, 0, 0, 0, 0, 0, 1], # 1x1 conv
[0, 0, 0, 0, 0, 0, 1], # 3x3 conv
[0, 0, 0, 0, 1, 0, 0], # 5x5 conv (replaced by two 3x3's)
[0, 0, 0, 0, 0, 0, 1], # 5x5 conv (replaced by two 3x3's)
[0, 0, 0, 0, 0, 0, 1], # 3x3 max-pool
[0, 0, 0, 0, 0, 0, 0]] # output layer
# Operations at the vertices of the module, matches order of matrix
labels=['input', 'conv1x1-bn-relu', 'conv3x3-bn-relu', 'conv3x3-bn-relu', 'conv3x3-bn-relu', 'maxpool3x3', 'output']
model_dir = '../my_model_dir'
print(tf.train.latest_checkpoint(model_dir))
sess = tf.Session()
saver = tf.train.import_meta_graph(tf.train.latest_checkpoint(model_dir)+'.meta')
saver.restore(sess, tf.train.latest_checkpoint(model_dir))
print('OK')
def main():
config = _config.build_config()
config['use_tpu'] = False
config['train_data_files'] = ['train_1.tfrecords', 'train_2.tfrecords', 'train_3.tfrecords', 'train_4.tfrecords']
config['valid_data_file'] = ['validation.tfrecords']
config['test_data_file'] = ['test.tfrecords']
config['sample_data_file'] = ['sample.tfrecords']
config['train_epochs'] = 1
config['use_KD'] = False
matrix=[[0, 1, 1, 1, 0, 1, 0], # input layer
[0, 0, 0, 0, 0, 0, 1], # 1x1 conv
[0, 0, 0, 0, 0, 0, 1], # 3x3 conv
[0, 0, 0, 0, 1, 0, 0], # 5x5 conv (replaced by two 3x3's)
[0, 0, 0, 0, 0, 0, 1], # 5x5 conv (replaced by two 3x3's)
[0, 0, 0, 0, 0, 0, 1], # 3x3 max-pool
[0, 0, 0, 0, 0, 0, 0]] # output layer
# Operations at the vertices of the module, matches order of matrix
labels=['input', 'conv1x1-bn-relu', 'conv3x3-bn-relu', 'conv3x3-bn-relu', 'conv3x3-bn-relu', 'maxpool3x3', 'output']
matrix = np.array(matrix)
labels = np.array(labels)
spec = model_spec.ModelSpec(matrix, labels)
model_dir = '../data/tmp'
meta = evaluate.train_and_evaluate(spec, config, model_dir)
output_file = os.path.join(model_dir, RESULTS_FILE)
with tf.gfile.Open(output_file, 'w') as f:
json.dump(meta, f, cls=NumpyEncoder)
print('OK')
print(spec.__dict__)
def network_fn(args):
genotype = arch_to_genotype(args.arch, args.policy)
if args.policy.lower() == 'nasbench':
model = NasBenchNet(3, genotype, _config.build_config())
elif args.policy.lower() in ['nao', 'darts', 'enas', 'random']:
CIFAR_CLASSES = 10
model = NetworkCIFAR(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
else:
raise NotImplementedError
return model, genotype
def __init__(self,
models_file,
output_dir,
worker_id=0,
total_workers=1,
model_id_regex='^',
sort_keys=False):
self.config = _config.build_config()
with tf.io.gfile.GFile(models_file) as f:
self.models = json.load(f)
self.remainders = None
self.ordered_keys = None
if FLAGS.remainders_file:
# Run only the modules and repeat numbers specified
with tf.io.gfile.GFile(FLAGS.remainders_file) as f:
self.remainders = json.load(f)
self.remainders = sorted(self.remainders)
self.num_models = len(self.remainders)
self.total_work_units = self.num_models
else:
# Filter keys to only those that fit the regex and order them so all
# workers see a canonical ordering.
regex = re.compile(model_id_regex)
evaluated_keys = [
key for key in self.models.keys() if regex.match(key)
]
if sort_keys:
self.ordered_keys = sorted(evaluated_keys)
else:
self.ordered_keys = evaluated_keys
self.num_models = len(self.ordered_keys)
self.total_work_units = self.num_models * self.config['num_repeats']
self.total_workers = total_workers
# If the worker is recovering from a restart, figure out where to restart
worker_recovery_dir = os.path.join(output_dir, '_recovery')
tf.io.gfile.makedirs(
worker_recovery_dir) # Silently succeeds if exists
self.recovery_file = os.path.join(worker_recovery_dir, str(worker_id))
if tf.io.gfile.exists(self.recovery_file):
with tf.io.gfile.GFile(self.recovery_file) as f:
self.current_index = int(f.read())
else:
self.current_index = worker_id
with tf.io.gfile.GFile(self.recovery_file, 'w') as f:
f.write(str(self.current_index))
assert self.current_index % self.total_workers == worker_id
self.output_dir = output_dir
def main(_):
config = _config.build_config()
# The default settings in config are exactly what was used to generate the
# dataset of models. However, given more epochs and a different learning rate
# schedule, it is possible to get higher accuracy.
config['train_epochs'] = 200
config['lr_decay_method'] = 'STEPWISE'
config['train_seconds'] = -1 # Disable training time limit
spec = create_best_nasbench_spec(config)
data = evaluate.augment_and_evaluate(spec, config, FLAGS.model_dir)
tf.compat.v1.logging.info(data)
def __init__(self, dataset_file):
"""Initialize dataset, this should only be done once."""
# TODO(chrisying): download the file directly if dataset_file not provided
self.config = config.build_config()
print('Loading dataset from file... This may take a few minutes...')
start = time.time()
self.dataset = {}
for serialized_row in tf.python_io.tf_record_iterator(dataset_file):
data_point = {}
row = json.loads(serialized_row)
module_hash = row[0]
metrics = model_metrics_pb2.ModelMetrics.FromString(
row[3].decode('base64'))
# TODO(chrisying): is it useful to keep the original adjacency and
# operations actually evaluated? Leaving this commented out because it
# saves a lot of memory.
'''
module_adjacency = row[1]
module_operations = row[2]
adjacency = np.array(map(int, list(metrics['module_adjacency'])))
dim = int(math.sqrt(len(adjacency)))
assert dim * dim == len(adjacency)
data_point['module_adjacency'] = adjacency.reshape((dim, dim))
data_point['module_operations'] = metrics['module_operations'].split(',')
'''
data_point['trainable_parameters'] = metrics.trainable_parameters
final_evaluation = metrics.evaluation_data[-1]
data_point['training_time'] = final_evaluation.training_time
data_point['train_accuracy'] = final_evaluation.train_accuracy
data_point[
'validation_accuracy'] = final_evaluation.validation_accuracy
data_point['test_accuracy'] = final_evaluation.test_accuracy
# TODO(chrisying): when we support multiple epoch lengths, dict key will
# include the epoch count as well
if module_hash not in self.dataset:
self.dataset[module_hash] = []
self.dataset[module_hash].append(data_point)
elapsed = time.time() - start
print('Loaded dataset in %d seconds' % elapsed)
self.history = {}
self.total_time_spent = 0
self.total_epochs_spent = 0
def main(*args, **kwargs):
nasbench = nasbench_api.NASBench(FLAGS.path_to_nasbench)
module = nasbench.fixed_statistics[FLAGS.hash_key]
spec = model_spec.ModelSpec(module['module_adjacency'], module['module_operations'])
config = nasbench_config.build_config()
for flag in FLAGS.flags_by_module_dict()[args[0][0]]:
config[flag.name] = flag.value
config['use_tpu'] = False
config['use_KD'] = False
config['intermediate_evaluations'] = ['1.0']
trainset_multipier = FLAGS.trainset_part_percentage / 100.0
config['num_train'] = int(config['num_train'] * trainset_multipier)
config['num_train_eval'] = int(config['num_train_eval'] * trainset_multipier)
config['num_augment'] = int(config['num_augment'] * trainset_multipier)
logging.info("Train and evaluate with config\n{}\n and spec\n{}".format(config, spec))
train(spec, config, FLAGS.save_path)
def __init__(self, dataset_file, seed=None):
"""Initialize dataset, this should only be done once per experiment.
Args:
dataset_file: path to .tfrecord file containing the dataset.
seed: random seed used for sampling queried models. Two NASBench objects
created with the same seed will return the same data points when queried
with the same models in the same order. By default, the seed is randomly
generated.
"""
self.config = config.build_config()
random.seed(seed)
print('Loading dataset from file... This may take a few minutes...')
start = time.time()
# Stores the fixed statistics that are independent of evaluation (i.e.,
# adjacency matrix, operations, and number of parameters).
# hash --> metric name --> scalar
self.fixed_statistics = {}
# Stores the statistics that are computed via training and evaluating the
# model on CIFAR-10. Statistics are computed for multiple repeats of each
# model at each max epoch length.
# hash --> epochs --> repeat index --> metric name --> scalar
self.computed_statistics = {}
# Valid queriable epoch lengths. {4, 12, 36, 108} for the full dataset or
# {108} for the smaller dataset with only the 108 epochs.
self.valid_epochs = set()
for serialized_row in tf.python_io.tf_record_iterator(dataset_file):
# Parse the data from the data file.
module_hash, epochs, raw_adjacency, raw_operations, raw_metrics = (
json.loads(serialized_row.decode('utf-8')))
dim = int(np.sqrt(len(raw_adjacency)))
adjacency = np.array([int(e) for e in list(raw_adjacency)],
dtype=np.int8)
adjacency = np.reshape(adjacency, (dim, dim))
operations = raw_operations.split(',')
metrics = model_metrics_pb2.ModelMetrics.FromString(
base64.b64decode(raw_metrics))
if module_hash not in self.fixed_statistics:
# First time seeing this module, initialize fixed statistics.
new_entry = {}
new_entry['module_adjacency'] = adjacency
new_entry['module_operations'] = operations
new_entry[
'trainable_parameters'] = metrics.trainable_parameters
self.fixed_statistics[module_hash] = new_entry
self.computed_statistics[module_hash] = {}
self.valid_epochs.add(epochs)
if epochs not in self.computed_statistics[module_hash]:
self.computed_statistics[module_hash][epochs] = []
# Each data_point consists of the metrics recorded from a single
# train-and-evaluation of a model at a specific epoch length.
data_point = {}
# Note: metrics.evaluation_data[0] contains the computed metrics at the
# start of training (step 0) but this is unused by this API.
# Evaluation statistics at the half-way point of training
half_evaluation = metrics.evaluation_data[1]
data_point['halfway_training_time'] = half_evaluation.training_time
data_point[
'halfway_train_accuracy'] = half_evaluation.train_accuracy
data_point['halfway_validation_accuracy'] = (
half_evaluation.validation_accuracy)
data_point['halfway_test_accuracy'] = half_evaluation.test_accuracy
# Evaluation statistics at the end of training
final_evaluation = metrics.evaluation_data[2]
data_point['final_training_time'] = final_evaluation.training_time
data_point[
'final_train_accuracy'] = final_evaluation.train_accuracy
data_point['final_validation_accuracy'] = (
final_evaluation.validation_accuracy)
data_point['final_test_accuracy'] = final_evaluation.test_accuracy
self.computed_statistics[module_hash][epochs].append(data_point)
elapsed = time.time() - start
print('Loaded dataset in %d seconds' % elapsed)
self.history = {}
self.training_time_spent = 0.0
self.total_epochs_spent = 0
import networkx as nx
import torch
import torch.nn as nn
import numpy as np
import IPython
from torch.distributions import Categorical
from search_policies.cnn.search_space.nas_bench.model_search import NasBenchCellSearch
from search_policies.cnn.search_space.nas_bench.nasbench_api_v2 import ModelSpec_v2
from search_policies.cnn.search_space.nas_bench.model_builder import compute_vertex_channels
from nasbench.lib import config as _config
from search_policies.cnn.search_space.nas_bench.operations import conv_bn_relu, MixedVertexDARTS
from search_policies.cnn.search_space.nas_bench.model import NasBenchNet, NasBenchCell
import torch.nn.functional as F
NASBENCH_CONFIG = _config.build_config()
class ModelSpec_DARTS(ModelSpec_v2):
# This is a dynamic model spec for NasBenchNetSearchDarts.
# n = intermediate nodes.
alpha_topology = None
AVAILABLE_OPS = ['conv3x3-bn-relu', 'conv1x1-bn-relu', 'maxpool3x3']
# This is size [n + 1, n], [:-1, :] is as original matrix without input and output [1:-1, 1:-1].
# [-1, :] is used for zero operation
# to extract operation, use softmax([:, node_id]).
# m = num_ops
alpha_ops = None
# size [m, n]. sample m operations accordingly.
# Extract operation: softmax([:, node_id]).
def __init__(self, dataset_file, seed=None):
"""Initialize dataset, this should only be done once per experiment.
Args:
dataset_file: path to .tfrecord file containing the dataset.
seed: random seed used for sampling queried models. Two NASBench objects
created with the same seed will return the same data points when queried
with the same models in the same order. By default, the seed is randomly
generated.
"""
# TODO(chrisying): download the file directly if dataset_file not provided
self.config = config.build_config()
random.seed(seed)
print('Loading dataset from file... This may take a few minutes...')
start = time.time()
self.dataset = {}
self.valid_epochs = set()
for serialized_row in tf.python_io.tf_record_iterator(dataset_file):
data_point = {}
row = json.loads(serialized_row)
module_hash = row[0]
max_epochs = row[1]
self.valid_epochs.add(max_epochs)
metrics = model_metrics_pb2.ModelMetrics.FromString(
row[4].decode('base64'))
# TODO(chrisying): is it useful to keep the original adjacency and
# operations actually evaluated? Leaving this commented out because it
# saves a lot of memory.
'''
module_adjacency = row[2]
module_operations = row[3]
adjacency = np.array(map(int, list(metrics['module_adjacency'])))
dim = int(math.sqrt(len(adjacency)))
assert dim * dim == len(adjacency)
data_point['module_adjacency'] = adjacency.reshape((dim, dim))
data_point['module_operations'] = metrics['module_operations'].split(',')
'''
data_point['trainable_parameters'] = metrics.trainable_parameters
half_evaluation = metrics.evaluation_data[1]
final_evaluation = metrics.evaluation_data[2]
data_point['training_time'] = (half_evaluation.training_time,
final_evaluation.training_time)
data_point['train_accuracy'] = (half_evaluation.train_accuracy,
final_evaluation.train_accuracy)
data_point['validation_accuracy'] = (half_evaluation.validation_accuracy,
final_evaluation.validation_accuracy)
data_point['test_accuracy'] = (half_evaluation.test_accuracy,
final_evaluation.test_accuracy)
key = (module_hash, max_epochs)
if key not in self.dataset:
self.dataset[key] = []
self.dataset[key].append(data_point)
elapsed = time.time() - start
print('Loaded dataset in %d seconds' % elapsed)
self.history = {}
self.training_time_spent = 0
self.total_epochs_spent = 0
import numpy as np
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot as plt
import pandas as pd
import seaborn as sns
import tensorflow as tf
from nasbench import api
from nasbench.lib import model_spec, model_builder, config, cifar
nb = api.NASBench('nasbench_only108.tfrecord')
data = {'params': [], 'time': [], 'acc': [], 'flops': []}
cfg = config.build_config()
cfg['train_data_files'] = [
'nasbench/scripts/train_1.tfrecords', 'nasbench/scripts/train_2.tfrecords',
'nasbench/scripts/train_3.tfrecords', 'nasbench/scripts/train_4.tfrecords'
]
cfg['use_tpu'] = False
keys = list()
count = 0
for key in nb.hash_iterator():
keys.append(key)
count += 1
def keytotuple(key):
cur_network_data = nb.get_metrics_from_hash(key)
help='train and test best model at this budget point')
parser.add_argument('--verbose', type=int, default=1)
args = parser.parse_args()
epochs = args.epochs
verbose = args.verbose
val_split = 0.2
num_classes = 10
best = np.load('best_file.npy', allow_pickle=True).item()
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
y_train = to_categorical(y_train, num_classes)
y_test = to_categorical(y_test, num_classes)
best_enc = best[epochs][0]
model_config = config.build_config()
model_spec = ModelSpec(matrix=best_enc['module_adjacency'],
ops=best_enc['module_operations'],
data_format=model_config['data_format'])
batch_size = 250
# model_config['num_stacks'] = 0
print(model_config)
inputs = tf.keras.layers.Input(x_train.shape[1:], batch_size)
net_outputs = build_keras_model(model_spec, inputs,
model_spec.ops, model_config)
model = tf.keras.Model(inputs=inputs, outputs=net_outputs)
num_train_imgs = int(x_train.shape[0]*(1-val_split))
from nasbench import api
from nasbench.lib import graph_util, config
from aw_nas import utils
from aw_nas.utils.exception import expect
from aw_nas.common import SearchSpace
from aw_nas.rollout.base import BaseRollout
from aw_nas.controller.base import BaseController
from aw_nas.evaluator.base import BaseEvaluator
from aw_nas.rollout.compare import CompareRollout
from aw_nas.evaluator.arch_network import ArchEmbedder
from aw_nas.utils import DenseGraphConvolution, DenseGraphFlow
VERTICES = 7
MAX_EDGES = 9
_nasbench_cfg = config.build_config()
def _literal_np_array(arr):
if arr is None:
return None
return "np.array({})".format(np.array2string(arr, separator=",").replace("\n", " "))
class _ModelSpec(api.ModelSpec):
def __repr__(self):
return "_ModelSpec({}, {}; pruned_matrix={}, pruned_ops={})".format(
_literal_np_array(self.original_matrix), self.original_ops,
_literal_np_array(self.matrix), self.ops)
def hash_spec(self, *args, **kwargs):
return super(_ModelSpec, self).hash_spec(_nasbench_cfg["available_ops"])
if config['data_format'] == 'channels_last':
net = tf.reduce_mean(net, [1, 2])
elif config['data_format'] == 'channels_first':
net = tf.reduce_mean(net, [2, 3])
else:
raise ValueError('invalid data_format')
# Fully-connected layer to labels
logits = tf.layers.dense(inputs=net, units=config['num_labels'])
return logits
if __name__ == '__main__':
best = np.load('best_file.npy', allow_pickle=True).item()
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
best_36 = best[36][0]
model_spec = api.ModelSpec(matrix=best_36['module_adjacency'],
ops=best_36['module_operations'])
print(x_train.shape)
print(config.build_config())
print(best_36['module_adjacency'], best_36['module_operations'])
inputs = tf.placeholder(tf.float32, shape=x_train.shape, name='Input')
# inputs = InputLayer(input_shape=x_train.shape[1:])
# inputs = tf.data.Dataset.from_tensor_slices((x_train, y_train))
model = build_model(inputs, model_spec, config.build_config())
# with tf.Session() as sess:
# writer = tf.summary.FileWriter("model-output", sess.graph)
# print(sess.run(model))
# writer.close()
