def correlation_between_one_shot_nb(model_path, config, epoch):
if config['search_space'] == '1':
search_space = SearchSpace1()
elif config['search_space'] == '2':
search_space = SearchSpace2()
elif config['search_space'] == '3':
search_space = SearchSpace3()
else:
raise ValueError('Unknown search space')
model = DartsWrapper(
save_path=model_path,
seed=0,
batch_size=128,
grad_clip=5,
epochs=200,
num_intermediate_nodes=search_space.num_intermediate_nodes,
search_space=search_space,
cutout=False)
if 'random_ws' in model_path:
discrete = True
normalize = False
else:
discrete = False
normalize = True
model.load(epoch=epoch)
nb_test_errors = []
nb_valid_errors = []
one_shot_test_errors = []
for adjacency_matrix, ops, model_spec in search_space.generate_search_space_without_loose_ends(
):
if str(config['search_space']) == '1' or str(
config['search_space']) == '2':
adjacency_matrix_ss = np.delete(np.delete(adjacency_matrix, -2, 0),
-2, 0)
# Remove input, output and 5th node
ops_ss = ops[1:-2]
elif str(config['search_space']) == '3':
adjacency_matrix_ss = adjacency_matrix
# Remove input and output node
ops_ss = ops[1:-1]
else:
raise ValueError('Unknown search space')
one_shot_test_error = model.evaluate_test(
(adjacency_matrix_ss, ops_ss),
split='test',
discrete=discrete,
normalize=normalize)
one_shot_test_errors.extend(np.repeat(one_shot_test_error, 3))
# Query NASBench
data = nasbench.query(model_spec)
nb_test_errors.extend([1 - item['test_accuracy'] for item in data])
nb_valid_errors.extend(
[1 - item['validation_accuracy'] for item in data])
print('NB', nb_test_errors[-1], 'OS', one_shot_test_errors[-1],
'weights', model.model.arch_parameters())
correlation = np.corrcoef(one_shot_test_errors, nb_test_errors)[0, -1]
return correlation, nb_test_errors, nb_valid_errors, one_shot_test_errors
def search_space_id_to_obj(id):
if int(id) == 1:
return SearchSpace1()
elif int(id) == 2:
return SearchSpace2()
elif int(id) == 3:
return SearchSpace3()
else:
raise ValueError('Search space unknown.')
def correlation_between_one_shot_nb(model_path, config, epoch):
if config['search_space'] == '1':
search_space = SearchSpace1()
elif config['search_space'] == '2':
search_space = SearchSpace2()
elif config['search_space'] == '3':
search_space = SearchSpace3()
else:
raise ValueError('Unknown search space')
model = DartsWrapper(save_path=model_path, seed=0, batch_size=128, grad_clip=5, epochs=200,
num_intermediate_nodes=search_space.num_intermediate_nodes, search_space=search_space,
cutout=False)
discrete = True
normalize = False
model.load(epoch=epoch)
controller = torch.load(os.path.join(model_path, 'controller_epoch_{}.pt'.format(epoch)))
nb_test_errors = {'4': [], '12': [], '36': [], '108': []}
nb_valid_errors = {'4': [], '12': [], '36': [], '108': []}
one_shot_test_errors = []
for idx in range(100):
(adjacency_matrix_ss, ops_ss), _, _ = controller()
print(adjacency_matrix_ss, ops_ss)
one_shot_test_error = model.evaluate_test((adjacency_matrix_ss, ops_ss), split='test', discrete=discrete,
normalize=normalize)
one_shot_test_errors.extend(np.repeat(one_shot_test_error, 3))
# # Query NASBench
# Create nested list from numpy matrix
if str(config['search_space']) == '1' or str(config['search_space']) == '2':
adjacency_matrix_ss = upscale_to_nasbench_format(adjacency_matrix_ss)
# Remove input, output and 5th node
ops_ss.append(CONV1X1)
nasbench_adjacency_matrix = adjacency_matrix_ss.astype(np.int).tolist()
ops_ss.insert(0, INPUT)
ops_ss.append(OUTPUT)
# Assemble the model spec
model_spec = api.ModelSpec(
# Adjacency matrix of the module
matrix=nasbench_adjacency_matrix,
# Operations at the vertices of the module, matches order of matrix
ops=ops_ss)
for nb_epoch_budget in [4, 12, 36, 108]:
data = nasbench.query(model_spec=model_spec, epochs=nb_epoch_budget)
nb_test_errors[str(nb_epoch_budget)].extend([1 - item['test_accuracy'] for item in data])
nb_valid_errors[str(nb_epoch_budget)].extend([1 - item['validation_accuracy'] for item in data])
# print('NB', nb_test_errors[-1], 'OS', one_shot_test_errors[-1], 'weights', model.model.arch_parameters())
# correlation = np.corrcoef(one_shot_test_errors, nb_test_errors)[0, -1]
return None, nb_test_errors, nb_valid_errors, one_shot_test_errors
def main():
# Select the search space to search in
if args.search_space == '1':
search_space = SearchSpace1()
elif args.search_space == '2':
search_space = SearchSpace2()
elif args.search_space == '3':
search_space = SearchSpace3()
else:
raise ValueError('Unknown search space')
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(args.init_channels,
CIFAR_CLASSES,
args.layers,
criterion,
output_weights=args.output_weights,
steps=search_space.num_intermediate_nodes,
search_space=search_space)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True)
valid_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[split:num_train]),
pin_memory=True)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, args)
nasbench = None
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
# Save the one shot model architecture weights for later analysis
arch_filename = os.path.join(
args.save, 'one_shot_architecture_{}.obj'.format(epoch))
with open(arch_filename, 'wb') as filehandler:
numpy_tensor_list = []
for tensor in model.arch_parameters():
numpy_tensor_list.append(tensor.detach().cpu().numpy())
pickle.dump(numpy_tensor_list, filehandler)
# Save the entire one-shot-model
filepath = os.path.join(args.save,
'one_shot_model_{}.obj'.format(epoch))
torch.save(model.state_dict(), filepath)
logging.info('architecture')
logging.info(numpy_tensor_list)
# training
train_acc, train_obj = train(train_queue, valid_queue, model,
architect, criterion, optimizer, lr,
epoch)
logging.info('train_acc %f', train_acc)
# validation
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
logging.info('STARTING EVALUATION')
if nasbench is None:
nasbench = NasbenchWrapper(
dataset_file='/nasbench_data/nasbench_only108.tfrecord')
test, valid, runtime, params = naseval.eval_one_shot_model(
config=args.__dict__,
model=arch_filename,
nasbench_results=nasbench)
index = np.random.choice(list(range(3)))
logging.info(
'TEST ERROR: %.3f | VALID ERROR: %.3f | RUNTIME: %f | PARAMS: %d' %
(test[index], valid[index], runtime[index], params[index]))
if args.s3_bucket is not None:
for root, dirs, files in os.walk(args.save):
for f in files:
if 'one_shot_model' not in f:
path = os.path.join(root, f)
upload_to_s3(path, args.s3_bucket, path)
def main():
# Select the search space to search in
if args.search_space == '1':
search_space = SearchSpace1()
elif args.search_space == '2':
search_space = SearchSpace2()
elif args.search_space == '3':
search_space = SearchSpace3()
else:
raise ValueError('Unknown search space')
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(args.init_channels,
CIFAR_CLASSES,
args.layers,
criterion,
output_weights=args.output_weights,
steps=search_space.num_intermediate_nodes,
search_space=search_space)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True)
valid_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[split:num_train]),
pin_memory=True)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, args)
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
# Save the one shot model architecture weights for later analysis
filehandler = open(
os.path.join(args.save,
'one_shot_architecture_{}.obj'.format(epoch)), 'wb')
numpy_tensor_list = []
for tensor in model.arch_parameters():
numpy_tensor_list.append(tensor.detach().cpu().numpy())
pickle.dump(numpy_tensor_list, filehandler)
# Save the entire one-shot-model
filepath = os.path.join(args.save,
'one_shot_model_{}.obj'.format(epoch))
torch.save(model.state_dict(), filepath)
logging.info('architecture', numpy_tensor_list)
# training
train_acc, train_obj = train(train_queue, valid_queue, model,
architect, criterion, optimizer, lr,
epoch)
logging.info('train_acc %f', train_acc)
# validation
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
def main(args):
# Fill in with root output path
root_dir = os.getcwd()
print('root_dir', root_dir)
if args.save_dir is None:
save_dir = os.path.join(
root_dir, 'experiments/random_ws/ss_{}_{}_{}'.format(
time.strftime("%Y%m%d-%H%M%S"), args.search_space, args.seed))
else:
save_dir = args.save_dir
if not os.path.exists(save_dir):
os.makedirs(save_dir)
if args.eval_only:
assert args.save_dir is not None
# Dump the config of the run folder
with open(os.path.join(save_dir, 'config.json'), 'w') as fp:
json.dump(args.__dict__, fp)
log_format = '%(asctime)s %(message)s'
logging.basicConfig(stream=sys.stdout,
level=logging.INFO,
format=log_format,
datefmt='%m/%d %I:%M:%S %p')
fh = logging.FileHandler(os.path.join(save_dir, 'log.txt'))
fh.setFormatter(logging.Formatter(log_format))
logging.getLogger().addHandler(fh)
logging.info(args)
if args.search_space == '1':
search_space = SearchSpace1()
elif args.search_space == '2':
search_space = SearchSpace2()
elif args.search_space == '3':
search_space = SearchSpace3()
else:
raise ValueError('Unknown search space')
if args.benchmark == 'ptb':
raise ValueError('PTB not supported.')
else:
data_size = 25000
time_steps = 1
B = int(args.epochs * data_size / args.batch_size / time_steps)
if args.benchmark == 'cnn':
from optimizers.random_search_with_weight_sharing.darts_wrapper_discrete import DartsWrapper
model = DartsWrapper(
save_dir,
args.seed,
args.batch_size,
args.grad_clip,
args.epochs,
num_intermediate_nodes=search_space.num_intermediate_nodes,
search_space=search_space,
init_channels=args.init_channels,
cutout=args.cutout)
else:
raise ValueError('Benchmarks other cnn on cifar are not available')
searcher = Random_NAS(B, model, args.seed, save_dir)
logging.info('budget: %d' % (searcher.B))
if not args.eval_only:
searcher.run()
archs = searcher.get_eval_arch()
else:
np.random.seed(args.seed + 1)
archs = searcher.get_eval_arch(2)
logging.info(archs)
arch = ' '.join([str(a) for a in archs[0][0]])
with open('/tmp/arch', 'w') as f:
f.write(arch)
return arch
help='population size')
parser.add_argument('--sample_size',
default=10,
type=int,
nargs='?',
help='sample_size')
parser.add_argument('--seed', default=0, type=int, help='random seed')
args = parser.parse_args()
nasbench = api.NASBench(args.data_dir)
if args.search_space == "1":
search_space = SearchSpace1()
elif args.search_space == "2":
search_space = SearchSpace2()
elif args.search_space == "3":
search_space = SearchSpace3()
else:
raise ValueError('Unknown search space')
for seed in range(6):
np.random.seed(seed)
output_path = os.path.join(args.output_path, "regularized_evolution")
os.makedirs(os.path.join(output_path), exist_ok=True)
# Set random_seed
if args.algorithm == 'RE':
history = regularized_evolution(cycles=args.n_iters,
population_size=args.pop_size,
sample_size=args.sample_size)
else:
history = random_search(cycles=args.n_iters)
def eval_one_shot_model(config, model):
nasbench = NasbenchWrapper(
dataset_file=
'/home/darts_weight_sharing_analysis/cnn/bohb/src/nasbench_analysis/nasbench_data/108_e/nasbench_only108.tfrecord'
)
model_list = pickle.load(open(model, 'rb'))
alphas_mixed_op = model_list[0]
chosen_node_ops = softmax(alphas_mixed_op, axis=-1).argmax(-1)
node_list = [PRIMITIVES[i] for i in chosen_node_ops]
alphas_output = model_list[1]
alphas_inputs = model_list[2:]
if config['search_space'] == '1':
search_space = SearchSpace1()
num_inputs = list(search_space.num_parents_per_node.values())[3:-1]
parents_node_3, parents_node_4 = \
[get_top_k(softmax(alpha, axis=1), num_input) for num_input, alpha in zip(num_inputs, alphas_inputs)]
output_parents = get_top_k(softmax(alphas_output), num_inputs[-1])
parents = {
'0': [],
'1': [0],
'2': [0, 1],
'3': parents_node_3,
'4': parents_node_4,
'5': output_parents
}
node_list = [INPUT, *node_list, CONV1X1, OUTPUT]
elif config['search_space'] == '2':
search_space = SearchSpace2()
num_inputs = list(search_space.num_parents_per_node.values())[2:]
parents_node_2, parents_node_3, parents_node_4 = \
[get_top_k(softmax(alpha, axis=1), num_input) for num_input, alpha in zip(num_inputs[:-1], alphas_inputs)]
output_parents = get_top_k(softmax(alphas_output), num_inputs[-1])
parents = {
'0': [],
'1': [0],
'2': parents_node_2,
'3': parents_node_3,
'4': parents_node_4,
'5': output_parents
}
node_list = [INPUT, *node_list, CONV1X1, OUTPUT]
elif config['search_space'] == '3':
search_space = SearchSpace3()
num_inputs = list(search_space.num_parents_per_node.values())[2:]
parents_node_2, parents_node_3, parents_node_4, parents_node_5 = \
[get_top_k(softmax(alpha, axis=1), num_input) for num_input, alpha in zip(num_inputs[:-1], alphas_inputs)]
output_parents = get_top_k(softmax(alphas_output), num_inputs[-1])
parents = {
'0': [],
'1': [0],
'2': parents_node_2,
'3': parents_node_3,
'4': parents_node_4,
'5': parents_node_5,
'6': output_parents
}
node_list = [INPUT, *node_list, OUTPUT]
else:
raise ValueError('Unknown search space')
adjacency_matrix = search_space.create_nasbench_adjacency_matrix(parents)
# Convert the adjacency matrix in format for nasbench
adjacency_list = adjacency_matrix.astype(np.int).tolist()
model_spec = api.ModelSpec(matrix=adjacency_list, ops=node_list)
# Query nasbench
data = nasbench.query(model_spec)
valid_error, test_error, runtime, params = [], [], [], []
for item in data:
test_error.append(1 - item['test_accuracy'])
valid_error.append(1 - item['validation_accuracy'])
runtime.append(item['training_time'])
params.append(item['trainable_parameters'])
return test_error, valid_error, runtime, params
def get_configuration_space(
seed: Union[int, None] = None) -> CS.ConfigurationSpace:
return NASBench1shot1BaseBenchmark._get_configuration_space(
SearchSpace3(), seed)
def __init__(self,
data_path: Union[Path, str, None] = None,
rng: Union[np.random.RandomState, int, None] = None):
super(NASBench1shot1SearchSpace3Benchmark,
self).__init__(data_path=data_path, rng=rng)
self.search_space = SearchSpace3()
def main():
# Select the search space to search in
if args.search_space == '1':
search_space = SearchSpace1()
elif args.search_space == '2':
search_space = SearchSpace2()
elif args.search_space == '3':
search_space = SearchSpace3()
else:
raise ValueError('Unknown search space')
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(args.num_linear_layers,
args.init_channels,
CIFAR_CLASSES,
args.layers,
criterion,
output_weights=args.output_weights,
steps=search_space.num_intermediate_nodes,
search_space=search_space)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True)
valid_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[split:num_train]),
pin_memory=True)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, args)
# Read a random sample of architectures
archs = pickle.load(
open(
'/home/siemsj/projects/darts_weight_sharing_analysis/nasbench_analysis/architecture_inductive_bias/sampled_architectures_from_search_space_3.obj',
'rb'))
arch = archs[args.arch_idx]
arch_parameters = get_weights_from_arch(arch, model)
model._arch_parameters = arch_parameters
try:
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
# increase the cutout probability linearly throughout search
train_transform.transforms[
-1].cutout_prob = args.cutout_prob * epoch / (args.epochs - 1)
logging.info('epoch %d lr %e cutout_prob %e', epoch, lr,
train_transform.transforms[-1].cutout_prob)
# Save the one shot model architecture weights for later analysis
arch_filename = os.path.join(
args.save, 'one_shot_architecture_{}.obj'.format(epoch))
with open(arch_filename, 'wb') as filehandler:
numpy_tensor_list = []
for tensor in model.arch_parameters():
numpy_tensor_list.append(tensor.detach().cpu().numpy())
pickle.dump(numpy_tensor_list, filehandler)
# Save the entire one-shot-model
# filepath = os.path.join(args.save, 'one_shot_model_{}.obj'.format(epoch))
# torch.save(model.state_dict(), filepath)
logging.info('architecture', numpy_tensor_list)
# training
train_acc, train_obj = train(train_queue, valid_queue, model,
architect, criterion, optimizer, lr,
epoch)
logging.info('train_acc %f', train_acc)
# validation
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
logging.info('STARTING EVALUATION')
test, valid, runtime, params = naseval.eval_one_shot_model(
config=args.__dict__, model=arch_filename)
index = np.random.choice(list(range(3)))
logging.info(
'TEST ERROR: %.3f | VALID ERROR: %.3f | RUNTIME: %f | PARAMS: %d' %
(test[index], valid[index], runtime[index], params[index]))
except Exception as e:
logging.exception('message')
def understanding(model_path):
with open(os.path.join(model_path, 'config.json')) as fp:
config = json.load(fp)
config['search_space'] = '3'
if config['search_space'] == '1':
search_space = SearchSpace1()
elif config['search_space'] == '2':
search_space = SearchSpace2()
elif config['search_space'] == '3':
search_space = SearchSpace3()
else:
raise ValueError('Unknown search space')
model = DartsWrapper(save_path=model_path, seed=0, batch_size=128, grad_clip=5, epochs=200,
num_intermediate_nodes=search_space.num_intermediate_nodes, search_space=search_space,
cutout=False)
for adjacency_matrix, ops, model_spec in search_space.generate_search_space_without_loose_ends():
if str(config['search_space']) == '1' or str(config['search_space']) == '2':
adjacency_matrix_ss = np.delete(np.delete(adjacency_matrix, -2, 0), -2, 0)
# Remove input, output and 5th node
ops_ss = ops[1:-2]
elif str(config['search_space']) == '3':
adjacency_matrix_ss = adjacency_matrix
# Remove input and output node
ops_ss = ops[1:-1]
else:
raise ValueError('Unknown search space')
arch_parameters = model.get_weights_from_arch((adjacency_matrix_ss, ops_ss))
acces = []
means = []
variances = []
maximum = []
minimum = []
ranges = []
L1 = []
L2 = []
data = nasbench.query(model_spec)
print(data)
a = model.model.extract_sub(arch_parameters[0]).state_dict()
keys = [key for key in a if
'bn' not in key and 'se' not in key and 'classifier' not in key and 'weight' in key and len(
a[key].shape) == 4]
acces.append()
weights_list = [a[key].cpu().numpy() for key in keys]
means.append(np.mean([np.mean(weights) for weights in weights_list]))
variances.append(np.mean([np.var(weights) for weights in weights_list]))
maximum.append(np.mean([np.max(weights) for weights in weights_list]))
minimum.append(np.mean([np.min(weights) for weights in weights_list]))
ranges.append(np.mean([np.max(weights) - np.min(weights) for weights in weights_list]))
L2.append(np.mean([np.linalg.norm(weights) for weights in weights_list]))
L1.append(np.mean([np.abs(weights).mean() for weights in weights_list]))
print("arch parameters:")
print(arch_parameters)
print("model spec")
print(model_spec.matrix)
print(model_spec.ops)
# print('adjacency_matrix_ss:')
# print(adjacency_matrix_ss)
# print('ops_ss:')
# print(ops_ss)
print()
print(model.model)
def main():
if not 'debug' in args.save:
from nasbench_analysis import eval_darts_one_shot_model_in_nasbench as naseval
# Select the search space to search in
if args.search_space == '1':
search_space = SearchSpace1()
elif args.search_space == '2':
search_space = SearchSpace2()
elif args.search_space == '3':
search_space = SearchSpace3()
else:
raise ValueError('Unknown search space')
torch.set_num_threads(3)
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(args.init_channels,
CIFAR_CLASSES,
args.layers,
criterion,
output_weights=args.output_weights,
steps=search_space.num_intermediate_nodes,
search_space=search_space)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
if 'debug' in args.save:
split = args.batch_size
num_train = 2 * args.batch_size
train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True)
valid_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[split:num_train]),
pin_memory=True)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
analyzer = Analyzer(model, args)
architect = Architect(model, args)
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
# Save the one shot model architecture weights for later analysis
arch_filename = os.path.join(
args.save, 'one_shot_architecture_{}.obj'.format(epoch))
with open(arch_filename, 'wb') as filehandler:
numpy_tensor_list = []
for tensor in model.arch_parameters():
numpy_tensor_list.append(tensor.detach().cpu().numpy())
pickle.dump(numpy_tensor_list, filehandler)
# # Save the entire one-shot-model
# filepath = os.path.join(args.save, 'one_shot_model_{}.obj'.format(epoch))
# torch.save(model.state_dict(), filepath)
for i in numpy_tensor_list:
print(i)
# training
train_acc, train_obj, ev = train(train_queue, valid_queue, model,
architect, criterion, optimizer, lr,
epoch, analyzer)
logging.info('train_acc %f', train_acc)
logging.info('eigenvalue %f', ev)
writer.add_scalar('Acc/train', train_acc, epoch)
writer.add_scalar('Obj/train', train_obj, epoch)
writer.add_scalar('Analysis/eigenvalue', ev, epoch)
# validation
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
writer.add_scalar('Acc/valid', valid_acc, epoch)
writer.add_scalar('Obj/valid', valid_obj, epoch)
utils.save(model, os.path.join(args.save, 'weights.pt'))
if not 'debug' in args.save:
# benchmark
logging.info('STARTING EVALUATION')
test, valid, runtime, params = naseval.eval_one_shot_model(
config=args.__dict__, model=arch_filename)
index = np.random.choice(list(range(3)))
test, valid, runtime, params = np.mean(test), np.mean(
valid), np.mean(runtime), np.mean(params)
logging.info(
'TEST ERROR: %.3f | VALID ERROR: %.3f | RUNTIME: %f | PARAMS: %d'
% (test, valid, runtime, params))
writer.add_scalar('Analysis/test', test, epoch)
writer.add_scalar('Analysis/valid', valid, epoch)
writer.add_scalar('Analysis/runtime', runtime, epoch)
writer.add_scalar('Analysis/params', params, epoch)
writer.close()