def analysis():
search_space_1 = SearchSpace1()
search_space_1.sample(with_loose_ends=False)
# Load NASBench
nasbench = NasbenchWrapper('nasbench_analysis/nasbench_data/108_e/nasbench_full.tfrecord')
test_error = []
valid_error = []
search_space_1 = SearchSpace1()
search_space_1.sample_with_loose_ends()
for i in range(10000):
adjacency_matrix, node_list = search_space_1.sample()
adjacency_list = adjacency_matrix.astype(np.int).tolist()
node_list = [INPUT, *node_list, OUTPUT]
model_spec = api.ModelSpec(matrix=adjacency_list, ops=node_list)
nasbench.query(model_spec)
for adjacency_matrix, ops, model_spec in search_space_1.generate_search_space_without_loose_ends():
# Query NASBench
data = nasbench.query(model_spec)
for item in data:
test_error.append(1 - item['test_accuracy'])
valid_error.append(1 - item['validation_accuracy'])
print('Number of architectures', len(test_error) / len(data))
plt.figure()
plt.title(
'Distribution of test error in search space (no. architectures {})'.format(
int(len(test_error) / len(data))))
plt.hist(test_error, bins=800, density=True)
ax = plt.gca()
ax.set_xscale('log')
ax.set_yscale('log')
plt.xlabel('Test error')
plt.grid(True, which="both", ls="-", alpha=0.5)
plt.tight_layout()
plt.xlim(0, 0.3)
plt.savefig('nasbench_analysis/search_spaces/export/search_space_1/test_error_distribution.pdf', dpi=600)
plt.show()
plt.figure()
plt.title('Distribution of validation error in search space (no. architectures {})'.format(
int(len(valid_error) / len(data))))
plt.hist(valid_error, bins=800, density=True)
ax = plt.gca()
ax.set_xscale('log')
ax.set_yscale('log')
plt.xlabel('Validation error')
plt.grid(True, which="both", ls="-", alpha=0.5)
plt.tight_layout()
plt.xlim(0, 0.3)
plt.savefig('nasbench_analysis/search_spaces/export/search_space_1/valid_error_distribution.pdf', dpi=600)
plt.show()
print('test_error', min(test_error), 'valid_error', min(valid_error))
def eval_directory(path):
"""Evaluates all one-shot architecture methods in the directory."""
# Read in config
nasbench = NasbenchWrapper(
dataset_file='/results/nasbench_only108.tfrecord')
with open(os.path.join(path, 'config.json')) as fp:
config = json.load(fp)
# Accumulate all one-shot models
one_shot_architectures = glob.glob(
os.path.join(path, 'one_shot_architecture_*.obj'))
# Sort them by date
one_shot_architectures.sort(key=natural_keys)
# Eval all of them
test_errors = []
valid_errors = []
for model in one_shot_architectures:
test, valid, _, _ = eval_one_shot_model(config=config, model=model)
test_errors.append(test)
valid_errors.append(valid)
with open(os.path.join(path, 'one_shot_validation_errors.obj'),
'wb') as fp:
pickle.dump(valid_errors, fp)
with open(os.path.join(path, 'one_shot_test_errors.obj'), 'wb') as fp:
pickle.dump(test_errors, fp)
def main():
# Select the search space to search in
if args.search_space == '1':
search_space = SearchSpace1()
elif args.search_space == '2':
search_space = SearchSpace2()
elif args.search_space == '3':
search_space = SearchSpace3()
else:
raise ValueError('Unknown search space')
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(args.init_channels,
CIFAR_CLASSES,
args.layers,
criterion,
output_weights=args.output_weights,
steps=search_space.num_intermediate_nodes,
search_space=search_space)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True)
valid_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[split:num_train]),
pin_memory=True)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, args)
nasbench = None
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
# Save the one shot model architecture weights for later analysis
arch_filename = os.path.join(
args.save, 'one_shot_architecture_{}.obj'.format(epoch))
with open(arch_filename, 'wb') as filehandler:
numpy_tensor_list = []
for tensor in model.arch_parameters():
numpy_tensor_list.append(tensor.detach().cpu().numpy())
pickle.dump(numpy_tensor_list, filehandler)
# Save the entire one-shot-model
filepath = os.path.join(args.save,
'one_shot_model_{}.obj'.format(epoch))
torch.save(model.state_dict(), filepath)
logging.info('architecture')
logging.info(numpy_tensor_list)
# training
train_acc, train_obj = train(train_queue, valid_queue, model,
architect, criterion, optimizer, lr,
epoch)
logging.info('train_acc %f', train_acc)
# validation
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
logging.info('STARTING EVALUATION')
if nasbench is None:
nasbench = NasbenchWrapper(
dataset_file='/nasbench_data/nasbench_only108.tfrecord')
test, valid, runtime, params = naseval.eval_one_shot_model(
config=args.__dict__,
model=arch_filename,
nasbench_results=nasbench)
index = np.random.choice(list(range(3)))
logging.info(
'TEST ERROR: %.3f | VALID ERROR: %.3f | RUNTIME: %f | PARAMS: %d' %
(test[index], valid[index], runtime[index], params[index]))
if args.s3_bucket is not None:
for root, dirs, files in os.walk(args.save):
for f in files:
if 'one_shot_model' not in f:
path = os.path.join(root, f)
upload_to_s3(path, args.s3_bucket, path)
help='Multiplicative factor'
' accross budgets.')
parser.add_argument('--space', type=int, default=1, help='NASBench space')
parser.add_argument('--seed', type=int, default=1, help='Seed')
args = parser.parse_args()
min_budget = args.min_budget
max_budget = args.max_budget
eta = args.eta
args.working_directory = os.path.join(
args.working_directory,
"search_space_{}/run{}-seed{}".format(args.space, args.run_id, args.seed))
nasbench = NasbenchWrapper(
dataset_file=
'src/nasbench_analysis/nasbench_data/108_e/nasbench_only108.tfrecord')
if args.array_id == 1:
os.makedirs(args.working_directory, exist_ok=True)
NS = NameServer(run_id=args.run_id,
nic_name='eth0',
working_directory=args.working_directory)
ns_host, ns_port = NS.start()
# BOHB is usually so cheap, that we can
# affort to run a worker on the master node, too.
worker = worker(min_budget=min_budget,
max_budget=max_budget,
eta=eta,
def eval_one_shot_model(config, model):
nasbench = NasbenchWrapper(
dataset_file=
'/home/darts_weight_sharing_analysis/cnn/bohb/src/nasbench_analysis/nasbench_data/108_e/nasbench_only108.tfrecord'
)
model_list = pickle.load(open(model, 'rb'))
alphas_mixed_op = model_list[0]
chosen_node_ops = softmax(alphas_mixed_op, axis=-1).argmax(-1)
node_list = [PRIMITIVES[i] for i in chosen_node_ops]
alphas_output = model_list[1]
alphas_inputs = model_list[2:]
if config['search_space'] == '1':
search_space = SearchSpace1()
num_inputs = list(search_space.num_parents_per_node.values())[3:-1]
parents_node_3, parents_node_4 = \
[get_top_k(softmax(alpha, axis=1), num_input) for num_input, alpha in zip(num_inputs, alphas_inputs)]
output_parents = get_top_k(softmax(alphas_output), num_inputs[-1])
parents = {
'0': [],
'1': [0],
'2': [0, 1],
'3': parents_node_3,
'4': parents_node_4,
'5': output_parents
}
node_list = [INPUT, *node_list, CONV1X1, OUTPUT]
elif config['search_space'] == '2':
search_space = SearchSpace2()
num_inputs = list(search_space.num_parents_per_node.values())[2:]
parents_node_2, parents_node_3, parents_node_4 = \
[get_top_k(softmax(alpha, axis=1), num_input) for num_input, alpha in zip(num_inputs[:-1], alphas_inputs)]
output_parents = get_top_k(softmax(alphas_output), num_inputs[-1])
parents = {
'0': [],
'1': [0],
'2': parents_node_2,
'3': parents_node_3,
'4': parents_node_4,
'5': output_parents
}
node_list = [INPUT, *node_list, CONV1X1, OUTPUT]
elif config['search_space'] == '3':
search_space = SearchSpace3()
num_inputs = list(search_space.num_parents_per_node.values())[2:]
parents_node_2, parents_node_3, parents_node_4, parents_node_5 = \
[get_top_k(softmax(alpha, axis=1), num_input) for num_input, alpha in zip(num_inputs[:-1], alphas_inputs)]
output_parents = get_top_k(softmax(alphas_output), num_inputs[-1])
parents = {
'0': [],
'1': [0],
'2': parents_node_2,
'3': parents_node_3,
'4': parents_node_4,
'5': parents_node_5,
'6': output_parents
}
node_list = [INPUT, *node_list, OUTPUT]
else:
raise ValueError('Unknown search space')
adjacency_matrix = search_space.create_nasbench_adjacency_matrix(parents)
# Convert the adjacency matrix in format for nasbench
adjacency_list = adjacency_matrix.astype(np.int).tolist()
model_spec = api.ModelSpec(matrix=adjacency_list, ops=node_list)
# Query nasbench
data = nasbench.query(model_spec)
valid_error, test_error, runtime, params = [], [], [], []
for item in data:
test_error.append(1 - item['test_accuracy'])
valid_error.append(1 - item['validation_accuracy'])
runtime.append(item['training_time'])
params.append(item['trainable_parameters'])
return test_error, valid_error, runtime, params
# Eval all models on nasbench
test_errors = []
valid_errors = []
for model in random_ws_archs:
test, valid = eval_random_ws_model(config=config, model=model)
test_errors.append(test)
valid_errors.append(valid)
with open(os.path.join(path, 'one_shot_validation_errors.obj'),
'wb') as fp:
pickle.dump(valid_errors, fp)
with open(os.path.join(path, 'one_shot_test_errors.obj'), 'wb') as fp:
pickle.dump(test_errors, fp)
def main():
for directory in get_directory_list("experiments/enas/"):
try:
eval_directory(directory)
except Exception as e:
print('error', e, directory)
if __name__ == '__main__':
nasbench = NasbenchWrapper(
dataset_file=
'/home/ANONYMOUS/projects/darts_weight_sharing_analysis/nasbench_analysis/nasbench_data/108_e/nasbench_full.tfrecord'
)
main()
def analysis():
search_space_1 = SearchSpace(num_parents_per_node={
'0': 0,
'1': 1,
'2': 2,
'3': 2,
'4': 2,
'5': 2
},
search_space_number=1,
num_intermediate_nodes=4)
# Load NASBench
nasbench = NasbenchWrapper(
'/home/siemsj/projects/darts_weight_sharing_analysis/nasbench_full.tfrecord'
)
test_error = []
valid_error = []
search_space_creator = search_space_1.create_search_space(
with_loose_ends=False, upscale=False)
for adjacency_matrix, ops, model_spec in search_space_creator:
# Query NASBench
data = nasbench.query(model_spec)
for item in data:
test_error.append(1 - item['test_accuracy'])
valid_error.append(1 - item['validation_accuracy'])
print('Number of architectures', len(test_error) / len(data))
plt.figure()
plt.title(
'Distribution of test error in search space (no. architectures {})'.
format(int(len(test_error) / len(data))))
plt.hist(test_error, bins=800, density=True)
ax = plt.gca()
ax.set_xscale('log')
ax.set_yscale('log')
plt.xlabel('Test error')
plt.grid(True, which="both", ls="-", alpha=0.5)
plt.tight_layout()
plt.xlim(0, 0.3)
plt.savefig(
'nasbench_analysis/search_spaces/export/search_space_1/test_error_distribution.pdf',
dpi=600)
plt.show()
plt.figure()
plt.title(
'Distribution of validation error in search space (no. architectures {})'
.format(int(len(valid_error) / len(data))))
plt.hist(valid_error, bins=800, density=True)
ax = plt.gca()
ax.set_xscale('log')
ax.set_yscale('log')
plt.xlabel('Validation error')
plt.grid(True, which="both", ls="-", alpha=0.5)
plt.tight_layout()
plt.xlim(0, 0.3)
plt.savefig(
'nasbench_analysis/search_spaces/export/search_space_1/valid_error_distribution.pdf',
dpi=600)
plt.show()
print('test_error', min(test_error), 'valid_error', min(valid_error))
# Eval all of them
test_errors = []
valid_errors = []
for model in one_shot_architectures:
test, valid, _, _ = eval_one_shot_model(config=config, model=model)
test_errors.append(test)
valid_errors.append(valid)
with open(os.path.join(path, 'one_shot_validation_errors.obj'),
'wb') as fp:
pickle.dump(valid_errors, fp)
with open(os.path.join(path, 'one_shot_test_errors.obj'), 'wb') as fp:
pickle.dump(test_errors, fp)
def main():
directories = get_directory_list("experiments/inductive_bias/")
directories.sort(key=natural_keys)
for directory in directories:
try:
eval_directory(directory)
except Exception as e:
print('error', e, directory)
if __name__ == '__main__':
nasbench = NasbenchWrapper(
dataset_file='/results/nasbench_only108.tfrecord')
main()
valid_errors = []
for model in one_shot_architectures:
test, valid, _, _ = eval_one_shot_model(config=config,
model=model,
nasbench=nasbench)
test_errors.append(test)
valid_errors.append(valid)
with open(os.path.join(path, 'one_shot_validation_errors.obj'),
'wb') as fp:
pickle.dump(valid_errors, fp)
with open(os.path.join(path, 'one_shot_test_errors.obj'), 'wb') as fp:
pickle.dump(test_errors, fp)
def main(nasbench):
directories = get_directory_list("experiments_2/random_ws/")
directories.sort(key=natural_keys)
for directory in directories:
try:
eval_directory(directory, nasbench)
except Exception as e:
print('error', e, directory)
if __name__ == '__main__':
nasbench = NasbenchWrapper(
dataset_file='/home/ydingau/data/nasbench_only108.tfrecord')
main(nasbench=nasbench)
"""Evaluates all one-shot architecture methods in the directory."""
# Read in config
with open(os.path.join(path, 'config.json')) as fp:
config = json.load(fp)
# Accumulate all one-shot models
# random_ws_archs = glob.glob(os.path.join(path, 'full_val_architecture_epoch_*.obj'))
random_ws_archs = glob.glob(os.path.join(path, file))
# Sort them by date
random_ws_archs.sort(key=natural_keys)
# Eval all models on nasbench
test_errors = []
valid_errors = []
for model in random_ws_archs:
result = eval_random_ws_model(config=config, model=model)
with open(os.path.join(path, 'local_search_result.obj'), 'wb') as fp:
pickle.dump(result, fp)
def main():
eval_directory('experiments/ft0_nov29/', file='local_search_300.obj')
if __name__ == '__main__':
nasbench = NasbenchWrapper(
dataset_file=
'/home/ubuntu/nas_benchmark_datasets/nasbench_full.tfrecord')
main()
print("arch parameters:")
print(arch_parameters)
print("model spec")
print(model_spec.matrix)
print(model_spec.ops)
# print('adjacency_matrix_ss:')
# print(adjacency_matrix_ss)
# print('ops_ss:')
# print(ops_ss)
print()
print(model.model)
def main():
understanding(args.model_path)
# Load NASBench
# eval_directory_on_epoch(args.model_path, args.epoch)
parser = argparse.ArgumentParser("correlation_analysis")
parser.add_argument('--data', type=str, default='../data', help='location of the darts corpus')
parser.add_argument('--model_path', default="/Users/liqi17thu/Desktop/darts/search_space_1/search-baseline-20200623-134823-0-1",
help='Path to where the models are stored.')
parser.add_argument('--epoch', type=int, help='Epoch', default=108)
args = parser.parse_args()
if __name__ == '__main__':
nasbench = NasbenchWrapper('/Users/liqi17thu/Documents/GitHub/nasbench/nasbench_full.tfrecord')
main()
'wb') as fp:
pickle.dump(nb_valid_errors, fp)
with open(os.path.join(path, 'one_shot_test_errors_{}.obj'.format(epoch)),
'wb') as fp:
pickle.dump(one_shot_test_errors, fp)
def main():
# Load NASBench
eval_directory_on_epoch(args.model_path, args.epoch)
parser = argparse.ArgumentParser("correlation_analysis")
parser.add_argument('--data',
type=str,
default='../data',
help='location of the darts corpus')
parser.add_argument(
'--model_path',
default=
"experiments/darts/search_space_1/search-baseline-20190821-171946-0-1",
help='Path to where the models are stored.')
parser.add_argument('--epoch', type=int, help='Epoch')
args = parser.parse_args()
if __name__ == '__main__':
nasbench = NasbenchWrapper(
'nasbench_analysis/nasbench_data/108_e/nasbench_full.tfrecord')
main()
import os
import pickle
import numpy as np
import torch
import torch.nn.functional as F
from nasbench import api
from nasbench_analysis.search_spaces.search_space_1 import SearchSpace1
from nasbench_analysis.search_spaces.search_space_2 import SearchSpace2
from nasbench_analysis.search_spaces.search_space_3 import SearchSpace3
from nasbench_analysis.utils import get_top_k, INPUT, OUTPUT, CONV1X1, NasbenchWrapper, natural_keys
from optimizers.darts.genotypes import PRIMITIVES
nasbench = NasbenchWrapper(
dataset_file='/nfs/data/xiangning/data/nasbench_only108.tfrecord')
def softmax(weights, axis=-1):
return F.softmax(torch.Tensor(weights), axis).data.cpu().numpy()
def get_directory_list(path):
"""Find directory containing config.json files"""
directory_list = []
# return nothing if path is a file
if os.path.isfile(path):
return []
# add dir to directorylist if it contains .json files
if len([f for f in os.listdir(path) if f == 'config.json']) > 0:
directory_list.append(path)
with open(os.path.join(path, 'correlation_{}.obj'.format(epoch)), 'wb') as fp:
pickle.dump(correlations, fp)
print(os.path.join(path, 'correlation_{}.obj'.format(epoch)))
with open(os.path.join(path, 'nb_test_errors_{}.obj'.format(epoch)), 'wb') as fp:
pickle.dump(nb_test_errors, fp)
with open(os.path.join(path, 'nb_valid_errors_{}.obj'.format(epoch)), 'wb') as fp:
pickle.dump(nb_valid_errors, fp)
with open(os.path.join(path, 'one_shot_test_errors_{}.obj'.format(epoch)), 'wb') as fp:
pickle.dump(one_shot_test_errors, fp)
def main():
# Load NASBench
eval_directory_on_epoch(args.model_path, args.epoch)
parser = argparse.ArgumentParser("correlation_analysis")
parser.add_argument('--data', type=str, default='../data', help='location of the darts corpus')
parser.add_argument('--model_path', default="experiments/darts/search_space_1/search-baseline-20190821-171946-0-1",
help='Path to where the models are stored.')
parser.add_argument('--epoch', type=int, help='Epoch')
args = parser.parse_args()
if __name__ == '__main__':
nasbench = NasbenchWrapper('/home/ANONYMOUS/projects/darts_weight_sharing_analysis/nasbench_full.tfrecord')
main()
random_ws_archs = glob.glob(
os.path.join(path, 'full_val_architecture_epoch_*.obj'))
# Sort them by date
random_ws_archs.sort(key=natural_keys)
# Eval all models on nasbench
test_errors = []
valid_errors = []
for model in random_ws_archs:
test, valid = eval_random_ws_model(config=config, model=model)
test_errors.append(test)
valid_errors.append(valid)
with open(os.path.join(path, 'one_shot_validation_errors.obj'),
'wb') as fp:
pickle.dump(valid_errors, fp)
with open(os.path.join(path, 'one_shot_test_errors.obj'), 'wb') as fp:
pickle.dump(test_errors, fp)
def main():
for directory in get_directory_list("experiments/random_ws/"):
eval_directory(directory)
if __name__ == '__main__':
nasbench = NasbenchWrapper(
dataset_file=
'nasbench_analysis/nasbench_data/108_e/nasbench_full.tfrecord')
main()
