def test_model():
metamodel = MetaModel(hyperparameters)
metamodel.populate_with_nasnet_metacells()
drop_path_tracker = DropPathTracker(
hyperparameters.parameters['DROP_PATH_CHANCE'], 0, total_steps)
first_cell = CellDataHolder(
3, hyperparameters.parameters['TARGET_FILTER_DIMS'],
metamodel.cells[0], False, drop_path_tracker, 0.)
def get_model():
cell_input = tf.keras.Input(dataset.images_shape)
cell_output = tf.keras.layers.Conv2D(
hyperparameters.parameters['TARGET_FILTER_DIMS'], 1, 1,
'same')(cell_input)
cell_output = first_cell.build([cell_output, cell_output])
cell_output = cell_output[0]
cell_output = tf.keras.layers.Lambda(lambda x: tf.reduce_mean(
input_tensor=x, axis=[1, 2]))(cell_output)
cell_output = tf.keras.layers.Dropout(.5)(cell_output)
cell_output = tf.keras.layers.Dense(10)(cell_output)
model = tf.keras.Model(inputs=cell_input, outputs=cell_output)
optimizer = tf.keras.optimizers.Adam(
learning_rate=hyperparameters.
parameters['MAXIMUM_LEARNING_RATE'])
model.compile(optimizer=optimizer,
loss=tf.keras.losses.SparseCategoricalCrossentropy(
from_logits=True),
metrics=['accuracy'])
return model
accuracies = []
for i in range(cell_samples):
cell_model = get_model()
cell_model.fit(dataset.train_images,
dataset.train_labels,
shuffle=True,
batch_size=hyperparameters.parameters['BATCH_SIZE'],
epochs=1,
callbacks=[drop_path_tracker])
model_accuracies = []
for test_set_index in range(len(dataset.test_set_images)):
accuracy = cell_model.evaluate(
dataset.test_set_images[test_set_index],
dataset.test_set_labels[test_set_index])[-1]
print(
f'{dataset.test_set_names[test_set_index]} test set accuracy: {accuracy}'
)
model_accuracies.append(accuracy)
# accuracy = cell_model.evaluate(dataset.test_images, dataset.test_labels)[-1]
# accuracies.append(accuracy)
accuracies.append(model_accuracies)
tf.keras.backend.clear_session()
del cell_model
return accuracies, metamodel.get_embedding()
def eval_model(embedding = None, metamodel = None):
model = metamodel
if model is None:
model = MetaModel(hyperparameters)
if embedding is None:
model.populate_with_nasnet_metacells()
else:
model.populate_from_embedding(embedding)
model.build_model(dataset.images_shape)
model.evaluate(dataset, 1, dir_path)
model.save_metadata(dir_path)
model.save_model(dir_path)
model.generate_graph(dir_path)
model.clear_model()
tf.keras.backend.clear_session()
def test_accuracy_at_different_train_amounts():
dir_path = os.path.join(evo_dir, 'test_accuracy_epochs')
if not os.path.exists(dir_path):
os.makedirs(dir_path)
hyperparameters = Hyperparameters()
hyperparameters.parameters['POPULATION_SIZE'] = 32
hyperparameters.parameters['ROUNDS'] = 0
hyperparameters.parameters['TRAIN_EPOCHS'] = 1
hyperparameters.parameters['TRAIN_ITERATIONS'] = 16
dataset = ImageDataset.get_cifar10()
existing_sims = [
x for x in os.listdir(dir_path) if 'small' not in x and '.png' not in x
]
num_already_done = len(existing_sims)
num_remaining = hyperparameters.parameters[
'POPULATION_SIZE'] - num_already_done
total_todo = hyperparameters.parameters['POPULATION_SIZE']
population = []
for round_num in range(num_remaining):
print(
f'Evaluating model {round_num + 1 + num_already_done} of {total_todo}'
)
new_candidate = MetaModel(hyperparameters)
new_candidate.populate_with_nasnet_metacells()
new_candidate.model_name = 'evo_' + str(
time.time()
) # this is redone here since all models are initialized within microseconds of eachother for init population
new_candidate.build_model(dataset.images_shape)
new_candidate.evaluate(dataset)
new_candidate.save_model(dir_path)
# new_candidate.metrics.metrics['accuracy'].extend([x + round_num for x in range(4)])
new_candidate.save_metadata(dir_path)
population.append(new_candidate)
new_candidate.clear_model()
def train_nasnet_archs():
num_models = 16
def default_params(epochs: int) -> Hyperparameters:
params = Hyperparameters()
params.parameters['REDUCTION_EXPANSION_FACTOR'] = 2
params.parameters['SGDR_EPOCHS_PER_RESTART'] = epochs
params.parameters['TRAIN_ITERATIONS'] = epochs
params.parameters['MAXIMUM_LEARNING_RATE'] = 0.025
params.parameters['MINIMUM_LEARNING_RATE'] = 0.001
params.parameters['DROP_PATH_TOTAL_STEPS_MULTI'] = 1
params.parameters['BATCH_SIZE'] = 16
return params
def standard_params(epochs: int) -> Hyperparameters:
params = default_params(epochs)
params.parameters['TARGET_FILTER_DIMS'] = 32
params.parameters['CELL_STACKS'] = [6, 1]
params.parameters['CELL_LAYERS'] = 3
return params
def medium_params(epochs: int, filters=32) -> Hyperparameters:
params = default_params(epochs)
params.parameters['TARGET_FILTER_DIMS'] = filters
params.parameters['CELL_STACKS'] = [5, 1]
params.parameters['CELL_LAYERS'] = 3
return params
def small_params(epochs: int) -> Hyperparameters:
params = default_params(epochs)
params.parameters['TARGET_FILTER_DIMS'] = 24
params.parameters['CELL_STACKS'] = [3, 1]
params.parameters['CELL_LAYERS'] = 3
return params
def long_params() -> Hyperparameters:
params = default_params(16)
params.parameters['TARGET_FILTER_DIMS'] = 16
params.parameters['CELL_STACKS'] = [3, 1]
params.parameters['CELL_LAYERS'] = 2
params.parameters['CONCATENATE_ALL'] = False
params.parameters['GROUPS_PER_CELL'] = 7
return params
embeddings = []
np.random.seed(0)
for i in range(num_models):
m = MetaModel(default_params(0))
m.populate_with_nasnet_metacells()
embeddings.append(m.get_embedding())
np.random.seed(0)
long_embeddings = []
for i in range(num_models):
m = MetaModel(long_params())
m.populate_with_nasnet_metacells()
long_embeddings.append(m.get_embedding())
multi_model_test('zs_small_3x3_16e_24f',
num_models=num_models,
hparams=small_params(16),
emb_queue=embeddings)
multi_model_test('zs_small_3x3_32e_24f',
num_models=num_models,
hparams=small_params(32),
emb_queue=embeddings)
multi_model_test('zs_medium_5x3_16e_24f',
num_models=num_models,
hparams=medium_params(32),
emb_queue=embeddings)
multi_model_test('zs_medium_6x3_16e_32f',
num_models=num_models,
hparams=medium_params(16, 32),
emb_queue=embeddings)
multi_model_test('zs_standard_6x3_16e_32f',
num_models=num_models,
hparams=medium_params(16),
emb_queue=embeddings)
multi_model_test('zs_standard_6x3_32e_32f',
num_models=num_models,
hparams=medium_params(32),
emb_queue=embeddings)
def run_test(dir_name):
cell_samples = 16
base_population = 8
evolved_population = 24
mods = [
ObjectModifier.SizeModifier, ObjectModifier.PerspectiveModifier,
ObjectModifier.RotationModifier, ObjectModifier.ColorModifier
]
hyperparameters = Hyperparameters()
dir_path = os.path.join(evo_dir, dir_name)
results_path = os.path.join(dir_path, 'results.json')
if not os.path.exists(dir_path):
os.makedirs(dir_path)
# load dataset, or create a new one if one doesn't exist
dataset_exists = os.path.exists(dir_path) and 'dataset.npy' in os.listdir(
dir_path)
if not dataset_exists:
print('Generating dataset')
DatasetGenerator.build_task_dataset(20000, (32, 32),
10,
4,
2,
dir_path,
modifiers=mods,
max_depth_of_target=1)
dataset = DatasetGenerator.get_task_dataset(dir_path)
# load previous test results if they exist
data = {'embeddings': [], 'accuracies': []}
if os.path.exists(results_path):
with open(results_path, 'r') as fl:
data = json.load(fl)
def save_data():
with open(results_path, 'w+') as fl:
json.dump(data, fl, indent=4)
def get_average_accuracy(model_index: int, cell_index: int):
return np.mean(data['accuracies'][model_index][cell_index], axis=0)
existing_population_size = len(data['embeddings'])
remaining_base_population = 0 if existing_population_size > base_population else base_population - existing_population_size
remaining_evolved_population = evolved_population if existing_population_size < base_population else evolved_population - (
existing_population_size - base_population)
print(
f'Evaluating {remaining_base_population} base candidates ({base_population - remaining_base_population}/{base_population} done) '
f'and {remaining_evolved_population} evolved candidates ({evolved_population - remaining_evolved_population}/{evolved_population} done)'
)
for i in range(remaining_base_population):
print(
f'Evaluating candidates {i} of {remaining_base_population} base candidates'
)
metamodel = MetaModel(hyperparameters)
metamodel.populate_with_nasnet_metacells()
accuracies = test_model(metamodel, dataset, cell_samples)
data['embeddings'].append(metamodel.get_embedding())
data['accuracies'].append(accuracies)
save_data()
performances = [performance(x) for x in data['accuracies']]
def find_best_indexes():
best_performances = np.full(performances[0].shape,
1.,
dtype=np.float32)
best_indexes = np.zeros(performances[0].shape, dtype=np.int)
for performance_index, x in enumerate(performances):
for i, entry in enumerate(x):
if best_performances[i] > entry:
best_performances[i] = entry
best_indexes[i] = performance_index
return best_indexes
for i in range(remaining_evolved_population):
print(
f'Evaluating candidates {i} of {remaining_evolved_population} evolved candidates'
)
best_indexes = find_best_indexes()
print(f'best indexes: {best_indexes}')
combined_embeddings = combine_embeddings(
data['embeddings'][best_indexes[0]],
data['embeddings'][best_indexes[1]])
mutated_embeddings = mutate_cell_from_embedding(combined_embeddings, 0)
mutated_embeddings = mutate_cell_from_embedding(mutated_embeddings, 1)
metamodel = MetaModel(hyperparameters)
metamodel.populate_from_embedding(mutated_embeddings)
accuracies = test_model(metamodel, dataset, cell_samples)
data['embeddings'].append(metamodel.get_embedding())
data['accuracies'].append(accuracies)
performances.append(performance(accuracies))
save_data()