def deserialize(self, obj: dict) -> None:
# for block in obj['blocks']:
# item = MetaCell()
# item.deserialize(block)
# self.cells.append(item)
self.populate_from_embedding(obj['embedding'])
self.model_name = obj['model_name']
self.metrics = Metrics()
self.metrics.deserialize(obj['metrics'])
self.hyperparameters = Hyperparameters()
self.hyperparameters.deserialize(obj['hyperparameters'])
if 'parent_model_name' in obj:
self.parent_model_name = obj['parent_model_name']
else:
self.parent_model_name = ''
def test1():
defaults = {
'a': 'a1',
'b': {
'c': 'c1'
},
'd': 'd1',
}
non_hash = {'e': {'f': 'f1', 'g': 'g1'}}
inputs = {
'a': 'a2', # This overrides a hash default
'b': {
'c': 'c2'
}, # This overrides a hash default
'e': {
'g': 'g2'
} # This overrides a non-hash default
}
correct_integration = {
'a': 'a2',
'b': {
'c': 'c2'
},
'd': 'd1',
'e': {
'f': 'f1',
'g': 'g2',
}
}
correct_hash = {'a': 'a2', 'b': {'c': 'c2'}}
# Test integration of hps
hps = Hyperparameters(inputs, defaults, non_hash)
print_test_result(hps._integrated_hps, correct_integration,
'test 1a: simple nested hp integration with defaults')
# Test partitioning for hashing
print_test_result(hps.hps_to_hash, correct_hash,
'test 1b: hp partitioning for hashing')
# Test base case __getattr__ (no recursion necessary)
print_test_result(hps['a'], 'a2',
'test 1c: base __getattr__ (no recursion)')
# Test recursive __getattr__
print_test_result(hps['b:c'], 'c2', 'test 1d: recursive __getattr__')
# Test recursive __setattr__
hps['e:g'] = 'g2'
print_test_result(hps['e:g'], 'g2', 'test 1e: recursive __setattr__')
# Test recursive __setattr__ for originally non-existing item
# (hopefully this never happens in practice)
hps['e:h'] = 'h2'
print_test_result(hps['e:h'], 'h2',
'test 1f: recursive __setattr__, non-existing item')
def test3():
# Test integrate_hps using the sub/super application
# See docstring for integrate_hps
super_def_hps = {'a': 1, 'b': 2, 'c_hps': {'d': 3, 'e': 4}}
sub_def_hps = {'b': 5, 'c_hps': {'d': 6}}
correct_result = {'a': 1, 'b': 5, 'c_hps': {'d': 6, 'e': 4}}
result = Hyperparameters.integrate_hps(super_def_hps, sub_def_hps)
print_test_result(result, correct_result,
'test 3a: subclass/superclass hp integration')
print_test_result(
super_def_hps, {
'a': 1,
'b': 2,
'c_hps': {
'd': 3,
'e': 4
}
}, 'test 3b: integration effects on original superclass hps')
print_test_result(sub_def_hps, {
'b': 5,
'c_hps': {
'd': 6
}
}, 'test 3c: integration effects on original subclass hps')
def verify_mutations():
params = Hyperparameters()
params.parameters['TRAIN_ITERATIONS'] = 1
params.parameters['REDUCTION_EXPANSION_FACTOR'] = 2
dataset = ImageDataset.get_cifar10_reduced()
for i in range(50):
model = MetaModel(params)
model.populate_with_nasnet_metacells()
model.build_model(dataset.images_shape)
model.mutate()
model.mutate()
model.mutate()
model.clear_model()
tf.keras.backend.clear_session()
def test_benchmark_models():
dir_path = os.path.join(evo_dir, 'cell_evo_benchmarks_6')
results_path = os.path.join(dir_path, 'results.json')
mods = [
ObjectModifier.SizeModifier, ObjectModifier.PerspectiveModifier,
ObjectModifier.RotationModifier, ObjectModifier.ColorModifier
]
hyperparameters = Hyperparameters()
cell_samples = 8
if not os.path.exists(dir_path):
os.makedirs(dir_path)
# load dataset, or create a new one if one doesn't exist
dataset_exists = os.path.exists(dir_path) and 'dataset.npy' in os.listdir(
dir_path)
if not dataset_exists:
print('Generating dataset')
# size, dim, num_classes, vertices per object, objects per image,
DatasetGenerator.build_task_dataset(40000, (16, 16),
10,
4,
10,
dir_path,
modifiers=mods,
max_depth_of_target=1)
dataset = DatasetGenerator.get_task_dataset(dir_path)
embeddings = [
MetaModel.get_nasnet_embedding(),
MetaModel.get_s1_embedding(),
MetaModel.get_identity_embedding(),
MetaModel.get_m1_sep3_embedding(),
MetaModel.get_m1_sep7_embedding(),
MetaModel.get_m1_sep3_serial_embedding(),
]
data = {'embeddings': [], 'accuracies': []}
if os.path.exists(results_path):
with open(results_path, 'r') as fl:
data = json.load(fl)
def save_data():
with open(results_path, 'w+') as fl:
json.dump(data, fl, indent=4)
for e in embeddings:
metamodel = MetaModel(hyperparameters)
metamodel.populate_from_embedding(e)
accuracies = test_model(metamodel, dataset, cell_samples)
data['embeddings'].append(metamodel.get_embedding())
data['accuracies'].append(accuracies)
save_data()
performances = [performance(x) for x in data['accuracies']]
print(performances)
def print_test_result(result, correct, name, verbose=False):
if result == correct:
print('Passed: %s.' % name)
else:
print('Failed %s.' % name)
verbose = True
if verbose:
if isinstance(correct, str):
correct_str = correct
result_str = result
else:
correct_str = Hyperparameters.printable_str_from_dict(correct)
result_str = Hyperparameters.printable_str_from_dict(result)
print('\nCorrect result:\n%s' % correct_str)
print('\nHyperparameters result:\n%s' % result_str)
def __init__(self, hyperparameters: Hyperparameters = Hyperparameters()):
super().__init__()
self.cells: List[MetaCell] = [] # [NORMAL_CELL, REDUCTION_CELL]
self.hyperparameters = hyperparameters
self.model_name = 'evo_' + str(time.time())
self.parent_model_name = ''
self.metrics = Metrics()
self.fitness = 0.
self.keras_model: tf.keras.Model = None
self.keras_model_data: ModelDataHolder = None
def verify_load():
dir_path = os.path.join(evo_dir, 'test_load_v2')
if not os.path.exists(dir_path):
os.makedirs(dir_path)
params = Hyperparameters()
params.parameters['TRAIN_ITERATIONS'] = 1
params.parameters['REDUCTION_EXPANSION_FACTOR'] = 2
dataset = ImageDataset.get_cifar10_reduced()
for i in range(50):
model = MetaModel(params)
model.populate_with_nasnet_metacells()
model.build_model(dataset.images_shape)
model.save_model(dir_path)
model.save_metadata(dir_path)
model.clear_model()
tf.keras.backend.clear_session()
other_model = MetaModel.load(dir_path, model.model_name, True)
tf.keras.backend.clear_session()
def test4():
# Same as test1, but also tests wildcard functionality
defaults = {'a': 'a1', 'b': {'c': 'c1'}, 'd': 'd1'}
non_hash = {'e': {'f': 'f1', 'g': 'g1'}}
inputs = {
'_wild1': 'w11', # WILDCARD (no default, forced to hash)
'a': 'a2', # This overrides a hash default
'b': {
'c': 'c2'
}, # This overrides a hash default
'e': {
'g': 'g2', # This overrides a non-hash default
'_wild2': 'w21' # WILDCARD (no default, forced to hash)
},
}
correct_hash = {
'_wild1': 'w11',
'a': 'a2',
'b': {
'c': 'c2'
},
'e': {
'_wild2': 'w21'
},
}
correct_integration = {
'_wild1': 'w11',
'a': 'a2',
'b': {
'c': 'c2'
},
'd': 'd1',
'e': {
'f': 'f1',
'g': 'g2',
'_wild2': 'w21'
},
}
# Test integration of hps
hps = Hyperparameters(inputs, defaults, non_hash)
print_test_result(hps._integrated_hps, correct_integration,
'test 4a: integration with wildcard (_)')
print_test_result(hps.hps_to_hash, correct_hash,
'test 4b: ensure wildcards are hashed')
def cell_performance_test_1():
hyperparameters = Hyperparameters()
dataset = ImageDataset.get_cifar10()
def get_sorted(images, labels):
sorted_by_class = [[] for _ in range(10)]
for index in range(len(images)):
sorted_by_class[labels[index, 0]].append(images[index, :, :])
sorted_train = get_sorted(dataset.train_images, dataset.train_labels)
sorted_test = get_sorted(dataset.test_images, dataset.test_labels)
model = MetaModel(hyperparameters)
# model.populate_with_nasnet_metacells()
model.populate_from_embedding(MetaModel.get_nasnet_embedding())
# model.build_model(dataset.images_shape)
first_cell = CellDataHolder(3, 3, model.cells[0])
cell_input = tf.keras.Input(dataset.images_shape)
cell_output = first_cell.build([cell_input, cell_input])
cell_model = tf.keras.Model(inputs=cell_input, outputs=cell_output)
def gram_matrix(input_tensor):
result = tf.linalg.einsum('bijc,bijd->bcd', input_tensor, input_tensor)
input_shape = tf.shape(input_tensor)
num_locations = tf.cast(input_shape[1] * input_shape[2], tf.float32)
return result / (num_locations)
optimizer = tf.keras.optimizers.Adam(
learning_rate=hyperparameters.parameters['LEARNING_RATE'])
def loss(real_image, fake_image, output):
real_maximize = None # TODO: INNER PROUDCT
fake_minimize = None
def train_step(input_image_1, input_image_2):
with tf.GradientTape() as tape:
image_1_output = cell_model(input_image_1)
image_2_output = cell_model(input_image_2)
total_loss = loss(input_image_1,
input_image_2, image_1_output) + loss(
input_image_2, input_image_1, image_2_output)
gradient = tape.gradient(loss, cell_model.trainable_variables)
optimizer.apply_gradients(zip(gradient,
cell_model.trainable_variables))
def test_nth_in_dir(dir_name, n: int):
dir_path = os.path.join(evo_dir, dir_name)
data_path = os.path.join(dir_path, 'results.json')
with open(data_path, 'r') as fl:
data = json.load(fl)
performances = [performance(x) for x in data['accuracies']]
performances_with_indexes = [(performances[i], data['embeddings'][i])
for i in range(len(performances))]
num_cells = len(performances[0]) # should be 2
pwi_per_cell = [performances_with_indexes.copy() for i in range(num_cells)]
for i in range(num_cells):
pwi_per_cell[i].sort(key=lambda x: x[0][i])
selected_embeddings = [x[n][1] for x in pwi_per_cell]
combined_embeddings = combine_embeddings(selected_embeddings[0],
selected_embeddings[1])
print(combined_embeddings)
hyperparameters = Hyperparameters()
hyperparameters.parameters['TRAIN_EPOCHS'] = 2
hyperparameters.parameters['TRAIN_ITERATIONS'] = 16
# hyperparameters.parameters['SGDR_EPOCHS_PER_RESTART'] = hyperparameters.parameters['TRAIN_ITERATIONS'] * hyperparameters.parameters['TRAIN_EPOCHS'] #effectively makes SGDR into basic cosine annealing
dataset = ImageDataset.get_cifar10()
metamodel = MetaModel(hyperparameters)
metamodel.populate_from_embedding(combined_embeddings)
metamodel.build_model(dataset.images_shape)
metamodel.evaluate(dataset)
metamodel.save_metadata(dir_path)
metamodel.save_model(dir_path)
metamodel.clear_model()
def main():
parser = argparse.ArgumentParser(description='')
parser.add_argument("-ds", "--dataset", default="AmazonCat-13K", type=str, required=True)
parser.add_argument("-t", "--head_threshold", default=10000, type=int)
parser.add_argument("-gpu", "--device_num", default='0', type=str)
parser.add_argument("-train", "--is_train", default=1, type=int)
parser.add_argument("-ep", "--epochs", default=8, type=int)
parser.add_argument("-ft", "--fine_tune", default=0, type=int)
parser.add_argument("-from", "--ft_from", default=0, type=int)
args = parser.parse_args()
ft = (args.fine_tune == 1)
ft_from = args.ft_from
hypes = Hyperparameters(args.dataset)
ds_path = '../datasets/' + args.dataset
device_num = args.device_num
head_threshold = args.head_threshold
with open(ds_path+'/mlc2seq/train_heads_X-'+str(head_threshold), 'rb') as g:
trn_head_X = pkl.load(g)
with open(ds_path+'/mlc2seq/train_heads_Y-'+str(head_threshold), 'rb') as g:
trn_head_Y = pkl.load(g)
with open(ds_path+'/mlc2seq/test_heads_X-'+str(head_threshold), 'rb') as g:
test_head_X = pkl.load(g)
with open(ds_path+'/mlc2seq/test_heads_Y-'+str(head_threshold), 'rb') as g:
test_head_Y = pkl.load(g)
answer = smat.load_npz(ds_path+'/Y.tst.npz')
with open(ds_path+'/mlc2seq/heads-'+str(head_threshold), 'rb') as g:
heads = pkl.load(g)
output_dim = len(heads)
bert = BertClassifier(hypes, heads, head_threshold, device_num, ft, args.epochs, max_seq_len=256)
trn_X_path = ds_path+'/head_data/trn_X-' + str(head_threshold)
test_X_path = ds_path+'/head_data/test_X-' + str(head_threshold)
trn_X = load_data(trn_X_path, trn_head_X, bert)
test_X = load_data(test_X_path, test_head_X, bert)
print('Number of labels:', len(heads))
print('Number of trn instances:', len(trn_X))
if args.is_train:
if ft:
print('======================Start Fine-Tuning======================')
model_path = '../save_models/head_classifier/'+hypes.dataset+'/t-'+str(head_threshold)+'_ep-' + str(ft_from)+'/pytorch_model.bin'
bert.train(trn_X, trn_head_Y, test_X, test_head_Y, model_path, ft_from)
else:
print('======================Start Training======================')
bert.train(trn_X, trn_head_Y, test_X, test_head_Y)
# bert.save()
else:
model_path = '../save_models/head_classifier/'+args.dataset+'/t-'+str(head_threshold)+'_ep-'+str(ft_from)+'/pytorch_model.bin'
print('======================Start Testing======================')
bert.evaluate(test_X, test_head_Y, model_path)
def get_flops_for_cell_models_from_embeddings():
tf.compat.v1.disable_eager_execution()
dir_path = os.path.join(evo_dir, 'cell_evo_benchmarks')
dataset = DatasetGenerator.get_task_dataset(dir_path)
hyperparameters = Hyperparameters()
embeddings = [
MetaModel.get_nasnet_embedding(),
MetaModel.get_s1_embedding(),
MetaModel.get_identity_embedding(),
MetaModel.get_m1_sep3_embedding(),
MetaModel.get_m1_sep7_embedding()
]
flops = []
for e in embeddings:
e_flops = []
metamodel = MetaModel(hyperparameters)
metamodel.populate_from_embedding(e)
steps_per_epoch = math.ceil(
len(dataset.train_labels) /
metamodel.hyperparameters.parameters['BATCH_SIZE'])
total_steps = metamodel.hyperparameters.parameters[
'TRAIN_ITERATIONS'] * metamodel.hyperparameters.parameters[
'TRAIN_EPOCHS'] * steps_per_epoch
for meta_cell in metamodel.cells:
drop_path_tracker = DropPathTracker(
metamodel.hyperparameters.parameters['DROP_PATH_CHANCE'], 0,
total_steps)
first_cell = CellDataHolder(
3, metamodel.hyperparameters.parameters['TARGET_FILTER_DIMS'],
meta_cell, False, drop_path_tracker, 0.)
cell_model = build_cell_model(
first_cell, dataset.images_shape,
metamodel.hyperparameters.parameters['TARGET_FILTER_DIMS'],
metamodel.hyperparameters.parameters['MAXIMUM_LEARNING_RATE'])
e_flops.append(
get_flops_for_keras_model(cell_model, dataset.images_shape))
tf.keras.backend.clear_session()
del cell_model
flops.append(e_flops)
print(flops)
print(flops)
def test_model_accuracy_from_embedding(dir_name, embedding):
dir_path = os.path.join(evo_dir, dir_name)
# dataset = ImageDataset.get_cifar10_reduced()
dataset = ImageDataset.get_cifar10()
if not os.path.exists(dir_path):
os.makedirs(dir_path)
hyperparameters = Hyperparameters()
model = MetaModel(hyperparameters)
model.populate_from_embedding(embedding)
model.build_model(dataset.images_shape)
model.evaluate(dataset)
model.save_model(dir_path)
model.generate_graph(dir_path)
model.save_metadata(dir_path)
model.clear_model()
def test_accuracy_at_different_train_amounts():
dir_path = os.path.join(evo_dir, 'test_accuracy_epochs')
if not os.path.exists(dir_path):
os.makedirs(dir_path)
hyperparameters = Hyperparameters()
hyperparameters.parameters['POPULATION_SIZE'] = 32
hyperparameters.parameters['ROUNDS'] = 0
hyperparameters.parameters['TRAIN_EPOCHS'] = 1
hyperparameters.parameters['TRAIN_ITERATIONS'] = 16
dataset = ImageDataset.get_cifar10()
existing_sims = [
x for x in os.listdir(dir_path) if 'small' not in x and '.png' not in x
]
num_already_done = len(existing_sims)
num_remaining = hyperparameters.parameters[
'POPULATION_SIZE'] - num_already_done
total_todo = hyperparameters.parameters['POPULATION_SIZE']
population = []
for round_num in range(num_remaining):
print(
f'Evaluating model {round_num + 1 + num_already_done} of {total_todo}'
)
new_candidate = MetaModel(hyperparameters)
new_candidate.populate_with_nasnet_metacells()
new_candidate.model_name = 'evo_' + str(
time.time()
) # this is redone here since all models are initialized within microseconds of eachother for init population
new_candidate.build_model(dataset.images_shape)
new_candidate.evaluate(dataset)
new_candidate.save_model(dir_path)
# new_candidate.metrics.metrics['accuracy'].extend([x + round_num for x in range(4)])
new_candidate.save_metadata(dir_path)
population.append(new_candidate)
new_candidate.clear_model()
def main():
parser = argparse.ArgumentParser(description='')
parser.add_argument("-ds", "--dataset", default="AmazonCat-13K", type=str, required=True)
parser.add_argument("-t", "--tail_threshold", default=100, type=int)
parser.add_argument("-le", "--label_embs", default='elmo', type=str)
parser.add_argument("-gpu", "--device_num", default='1', type=str)
parser.add_argument("-train", "--is_train", default=1, type=int)
parser.add_argument("-ep", "--epochs", default=8, type=int)
parser.add_argument("-ft", "--fine_tune", default=0, type=int)
parser.add_argument("-from", "--ft_from", default=0, type=int)
args = parser.parse_args()
hypes = Hyperparameters(args.dataset)
ds_path = '../datasets/' + args.dataset
tail_threshold = args.tail_threshold
device_num = args.device_num
label_embs = args.label_embs
fine_tune = args.fine_tune
epochs = args.epochs
is_train = args.is_train
ft = (args.fine_tune == 1)
ft_from = args.ft_from
graphfile = ds_path+'/label_graph'
label_graph = nx.read_edgelist(graphfile, create_using=nx.Graph())
with open(ds_path+'/mlc2seq/heads-'+str(tail_threshold), 'rb') as g:
heads = pkl.load(g)
with open(ds_path+'/mlc2seq/train_heads_Y-'+str(tail_threshold), 'rb') as g:
trn_head_Y = pkl.load(g)
with open(ds_path+'/mlc2seq/test_heads_Y-'+str(tail_threshold), 'rb') as g:
test_head_Y = pkl.load(g)
answer = smat.load_npz(ds_path+'/Y.tst.npz')
with open(ds_path+'/mlc2seq/tails-'+str(tail_threshold), 'rb') as g:
tails = pkl.load(g)
bertReg = BertRegressor(hypes, tails, label_embs, tail_threshold, device_num, epochs, ft, max_seq_len=hypes.max_seq_len)
trn_X_path = ds_path+'/tail_data/trn_X-' + str(tail_threshold)
test_X_path = ds_path+'/tail_data/test_X-' + str(tail_threshold)
if label_embs == 'elmo' or label_embs == 'en':
label_space = smat.load_npz(ds_path+'/L.elmo.npz')
label_space = smat.lil_matrix(label_space)
label_space[:tails[0]] = 9999
trn_Y_path = ds_path+'/tail_data/trn_elmo_Y-' + str(tail_threshold)
test_Y_path = ds_path+'/tail_data/test_elmo_Y-' + str(tail_threshold)
with open(ds_path+'/mlc2seq/train_tails_X-'+str(tail_threshold), 'rb') as g:
trn_tail_X = pkl.load(g)
with open(ds_path+'/mlc2seq/train_tails_Y-'+str(tail_threshold), 'rb') as g:
trn_tail_Y = pkl.load(g)
trn_X, trn_Y, trn_Y_nums = load_data(trn_X_path, trn_Y_path, trn_tail_X, trn_tail_Y, bertReg, label_space, 'elmo')
del(trn_tail_X, trn_tail_Y)
with open(ds_path+'/mlc2seq/test_tails_X-'+str(tail_threshold), 'rb') as g:
test_tail_X = pkl.load(g)
with open(ds_path+'/mlc2seq/test_tails_Y-'+str(tail_threshold), 'rb') as g:
test_tail_Y = pkl.load(g)
test_X, test_Y, test_Y_nums = load_data(test_X_path, test_Y_path, test_tail_X, test_tail_Y, bertReg, label_space, 'elmo')
del(test_tail_X, test_tail_Y)
elif label_embs == 'n2v':
lv = KV.load_word2vec_format(ds_path+'/label_n2v_embedding', binary=False)
trn_Y_path = ds_path+'/tail_data/trn_n2v_Y-' + str(tail_threshold)
test_Y_path = ds_path+'/tail_data/test_n2v_Y-' + str(tail_threshold)
trn_X, trn_Y, trn_Y_nums = load_data(trn_X_path, trn_Y_path, trn_tail_X, trn_tail_Y, bertReg, lv, 'n2v')
label_space = smat.csr_matrix(np.matrix(trn_Y))
test_X, test_Y, test_Y_nums = load_data(test_X_path, test_Y_path, test_tail_X, test_tail_Y, bertReg, lv, 'n2v')
else:
print('invalid label embedding type')
exit()
print('Number of instnaces:', len(trn_X))
print('Number of tail labels:', len(tails))
if is_train:
if ft:
print('======================Start Fine-tuning======================')
model_path = '../save_models/tail_regressor/'+hypes.dataset+'/t-'+str(tail_threshold)+'_ep-' + str(ft_from)+'-elmo/pytorch_model.bin'
bertReg.train(trn_X, trn_Y, trn_Y_nums, label_space, test_X, test_Y, test_Y_nums, model_path, ft_from)
else:
print('======================Start Training======================')
bertReg.train(trn_X, trn_Y, trn_Y_nums, label_space, test_X, test_Y, test_Y_nums)
output_dir = '../save_models/tail_regressor/'+hypes.dataset+'/t-'+str(tail_threshold)+'_ep-' + str(epochs + ft_from) +'-'+ label_embs+'/'
bertReg.save(output_dir)
accs = bertReg.evaluate(test_X, test_Y, test_Y_nums, label_space)
else:
model_path = '../save_models/tail_regressor/' + hypes.dataset + '/t-' +str(tail_threshold)+'_ep-' + str(ft_from)+'-'+label_embs+'/pytorch_model.bin'
print('======================Start Testing======================')
accs = bertReg.evaluate(test_X, test_Y, test_Y_nums, label_space, model_path, ft_from)
def cell_performance_test_2():
dir_path = os.path.join(evo_dir, 'dataset_gen')
hyperparameters = Hyperparameters()
hyperparameters.parameters['TARGET_FILTER_DIMS'] = 64
mods = [
# ObjectModifier.SizeModifier,
# ObjectModifier.PerspectiveModifier,
# ObjectModifier.RotationModifier,
# ObjectModifier.ColorModifier
]
DatasetGenerator.build_task_dataset(20000, (32, 32),
10,
4,
2,
dir_path,
modifiers=mods,
max_depth_of_target=1)
dataset = DatasetGenerator.get_task_dataset(dir_path)
cell_samples = 50
num_cells = 1
steps_per_epoch = math.ceil(
len(dataset.train_labels) / hyperparameters.parameters['BATCH_SIZE'])
total_steps = hyperparameters.parameters[
'TRAIN_ITERATIONS'] * hyperparameters.parameters[
'TRAIN_EPOCHS'] * steps_per_epoch
def test_model():
metamodel = MetaModel(hyperparameters)
metamodel.populate_with_nasnet_metacells()
drop_path_tracker = DropPathTracker(
hyperparameters.parameters['DROP_PATH_CHANCE'], 0, total_steps)
first_cell = CellDataHolder(
3, hyperparameters.parameters['TARGET_FILTER_DIMS'],
metamodel.cells[0], False, drop_path_tracker, 0.)
def get_model():
cell_input = tf.keras.Input(dataset.images_shape)
cell_output = tf.keras.layers.Conv2D(
hyperparameters.parameters['TARGET_FILTER_DIMS'], 1, 1,
'same')(cell_input)
cell_output = first_cell.build([cell_output, cell_output])
cell_output = cell_output[0]
cell_output = tf.keras.layers.Lambda(lambda x: tf.reduce_mean(
input_tensor=x, axis=[1, 2]))(cell_output)
cell_output = tf.keras.layers.Dropout(.5)(cell_output)
cell_output = tf.keras.layers.Dense(10)(cell_output)
model = tf.keras.Model(inputs=cell_input, outputs=cell_output)
optimizer = tf.keras.optimizers.Adam(
learning_rate=hyperparameters.
parameters['MAXIMUM_LEARNING_RATE'])
model.compile(optimizer=optimizer,
loss=tf.keras.losses.SparseCategoricalCrossentropy(
from_logits=True),
metrics=['accuracy'])
return model
accuracies = []
for i in range(cell_samples):
cell_model = get_model()
cell_model.fit(dataset.train_images,
dataset.train_labels,
shuffle=True,
batch_size=hyperparameters.parameters['BATCH_SIZE'],
epochs=1,
callbacks=[drop_path_tracker])
model_accuracies = []
for test_set_index in range(len(dataset.test_set_images)):
accuracy = cell_model.evaluate(
dataset.test_set_images[test_set_index],
dataset.test_set_labels[test_set_index])[-1]
print(
f'{dataset.test_set_names[test_set_index]} test set accuracy: {accuracy}'
)
model_accuracies.append(accuracy)
# accuracy = cell_model.evaluate(dataset.test_images, dataset.test_labels)[-1]
# accuracies.append(accuracy)
accuracies.append(model_accuracies)
tf.keras.backend.clear_session()
del cell_model
return accuracies, metamodel.get_embedding()
all_accuracies = []
all_embeddings = []
for x in range(num_cells):
accuracies, embedding = test_model()
all_accuracies.append(accuracies)
all_embeddings.append(embedding)
all_accuracies = np.array(all_accuracies)
print(f'all accuracies shape: {all_accuracies.shape}')
for i in range(len(all_accuracies)):
print(
f'embedding: {all_embeddings[i]}, avgs: {np.mean(all_accuracies[i], axis=0)}'
)
output_path = os.path.join(dir_path, 'results.json')
data = {
'embeddings': all_embeddings,
'accuracies': all_accuracies.tolist()
}
with open(output_path, 'w+') as fl:
json.dump(data, fl, indent=4)
def run_test(dir_name):
cell_samples = 16
base_population = 8
evolved_population = 24
mods = [
ObjectModifier.SizeModifier, ObjectModifier.PerspectiveModifier,
ObjectModifier.RotationModifier, ObjectModifier.ColorModifier
]
hyperparameters = Hyperparameters()
dir_path = os.path.join(evo_dir, dir_name)
results_path = os.path.join(dir_path, 'results.json')
if not os.path.exists(dir_path):
os.makedirs(dir_path)
# load dataset, or create a new one if one doesn't exist
dataset_exists = os.path.exists(dir_path) and 'dataset.npy' in os.listdir(
dir_path)
if not dataset_exists:
print('Generating dataset')
DatasetGenerator.build_task_dataset(20000, (32, 32),
10,
4,
2,
dir_path,
modifiers=mods,
max_depth_of_target=1)
dataset = DatasetGenerator.get_task_dataset(dir_path)
# load previous test results if they exist
data = {'embeddings': [], 'accuracies': []}
if os.path.exists(results_path):
with open(results_path, 'r') as fl:
data = json.load(fl)
def save_data():
with open(results_path, 'w+') as fl:
json.dump(data, fl, indent=4)
def get_average_accuracy(model_index: int, cell_index: int):
return np.mean(data['accuracies'][model_index][cell_index], axis=0)
existing_population_size = len(data['embeddings'])
remaining_base_population = 0 if existing_population_size > base_population else base_population - existing_population_size
remaining_evolved_population = evolved_population if existing_population_size < base_population else evolved_population - (
existing_population_size - base_population)
print(
f'Evaluating {remaining_base_population} base candidates ({base_population - remaining_base_population}/{base_population} done) '
f'and {remaining_evolved_population} evolved candidates ({evolved_population - remaining_evolved_population}/{evolved_population} done)'
)
for i in range(remaining_base_population):
print(
f'Evaluating candidates {i} of {remaining_base_population} base candidates'
)
metamodel = MetaModel(hyperparameters)
metamodel.populate_with_nasnet_metacells()
accuracies = test_model(metamodel, dataset, cell_samples)
data['embeddings'].append(metamodel.get_embedding())
data['accuracies'].append(accuracies)
save_data()
performances = [performance(x) for x in data['accuracies']]
def find_best_indexes():
best_performances = np.full(performances[0].shape,
1.,
dtype=np.float32)
best_indexes = np.zeros(performances[0].shape, dtype=np.int)
for performance_index, x in enumerate(performances):
for i, entry in enumerate(x):
if best_performances[i] > entry:
best_performances[i] = entry
best_indexes[i] = performance_index
return best_indexes
for i in range(remaining_evolved_population):
print(
f'Evaluating candidates {i} of {remaining_evolved_population} evolved candidates'
)
best_indexes = find_best_indexes()
print(f'best indexes: {best_indexes}')
combined_embeddings = combine_embeddings(
data['embeddings'][best_indexes[0]],
data['embeddings'][best_indexes[1]])
mutated_embeddings = mutate_cell_from_embedding(combined_embeddings, 0)
mutated_embeddings = mutate_cell_from_embedding(mutated_embeddings, 1)
metamodel = MetaModel(hyperparameters)
metamodel.populate_from_embedding(mutated_embeddings)
accuracies = test_model(metamodel, dataset, cell_samples)
data['embeddings'].append(metamodel.get_embedding())
data['accuracies'].append(accuracies)
performances.append(performance(accuracies))
save_data()
def main():
parser = argparse.ArgumentParser(description='')
parser.add_argument("-ds",
"--dataset",
default="AmazonCat-13K",
type=str,
required=True)
# parser.add_argument("-t", "--head_threshold", default=10000, type=int)
parser.add_argument("-k", "--k", default=256, type=int)
parser.add_argument("-gpu", "--device_num", default='0', type=str)
parser.add_argument("-train", "--is_train", default=1, type=int)
parser.add_argument("-ep", "--epochs", default=8, type=int)
parser.add_argument("-ft", "--fine_tune", default=0, type=int)
parser.add_argument("-clu", "--clustering", default='spec', type=str)
parser.add_argument("-from", "--ft_from", default=0, type=int)
parser.add_argument("-dlf", "--dlf", default=1, type=int)
args = parser.parse_args()
ft = (args.fine_tune == 1)
dlf = (args.dlf == 1)
ft_from = args.ft_from
num_clusters = args.k
hypes = Hyperparameters(args.dataset)
ds_path = '../datasets/' + args.dataset
device_num = args.device_num
with open(ds_path + '/mlc2seq/train_clus_X', 'rb') as g:
trn_clus_X = pkl.load(g)
with open(ds_path + '/mlc2seq/train_clus_Y', 'rb') as g:
trn_clus_Y = pkl.load(g)
with open(ds_path + '/mlc2seq/test_clus_X', 'rb') as g:
test_clus_X = pkl.load(g)
with open(ds_path + '/mlc2seq/test_clus_Y', 'rb') as g:
test_clus_Y = pkl.load(g)
answer = smat.load_npz(ds_path + '/Y.tst.npz')
label_space = load_label(ds_path)
output_dim = label_space.shape[0]
gutil = GraphUtil(trn_clus_Y, output_dim)
adj = gutil.gen_graph()
# gutil.cal_degree()
lc = LabelCluster(adj, num_clusters)
# c_adj: k*k, C: m*k
print('partitioning labels with ' + args.clustering + ' clustering...')
clus_path = ds_path + '/clus_data/k-' + str(num_clusters)
if args.clustering == 'kmeans':
C, c_adj = lc.kmeans_clustering(trn_clus_Y, label_space, clus_path)
elif args.clustering == 'spec':
C, c_adj = lc.spec_clustering(trn_clus_Y, clus_path)
else:
C, c_adj = lc.spec_clustering(trn_clus_Y, clus_path)
bert = BertGCN_ClusterClassifier(hypes,
device_num,
ft,
args.epochs,
label_space,
C,
c_adj,
dlf,
max_seq_len=256)
trn_X_path = ds_path + '/clus_data/trn_X'
test_X_path = ds_path + '/clus_data/test_X'
trn_X = load_data(trn_X_path, trn_clus_X, bert)
test_X = load_data(test_X_path, test_clus_X, bert)
print('Number of labels:', output_dim)
print('Number of trn instances:', len(trn_X))
if args.is_train:
if ft:
print(
'======================Start Fine-Tuning======================'
)
model_path = '/mnt/sdb/yss/xbert_save_models/gcn_dlf/' + hypes.dataset + '/ep-' + str(
ft_from) + '/k-' + str(num_clusters) + '/pytorch_model.bin'
# model_path = '../save_models/gcn_dlf/'+hypes.dataset+'/ep-' + str(ft_from) + '/k-' + str(num_clusters) + '/pytorch_model.bin'
bert.train(trn_X, trn_clus_Y, test_X, test_clus_Y, model_path,
ft_from)
else:
print('======================Start Training======================')
bert.train(trn_X, trn_clus_Y, test_X, test_clus_Y)
bert.evaluate(test_X, test_clus_Y)
output_dir = '/mnt/sdb/yss/xbert_save_models/gcn_dlf/' + hypes.dataset + '/ep-' + str(
args.epochs + ft_from) + '/k-' + str(num_clusters) + '/'
# output_dir = '../save_models/gcn_dlf/'+hypes.dataset+'/ep-' + str(args.epochs + ft_from) + '/k-' + str(num_clusters) + '/'
bert.save(output_dir)
else:
import datetime
# model_path = '../save_models/gcn_dlf/'+hypes.dataset+'/ep-' + str(ft_from) + '/k-' + str(num_clusters) + '/pytorch_model.bin'
model_path = '/mnt/sdb/yss/xbert_save_models/gcn_dlf/' + hypes.dataset + '/ep-' + str(
ft_from) + '/k-' + str(num_clusters) + '/pytorch_model.bin'
print('======================Start Testing======================')
start = datetime.datetime.now()
bert.evaluate(test_X, test_clus_Y, model_path)
end = datetime.datetime.now()
elapsed = end - start
print(elapsed.seconds, ":", elapsed.microseconds)
device = None
if USE_GPU and torch.cuda.is_available():
device = torch.device('cuda')
else:
device = torch.device('cpu')
import numpy as np
import matplotlib.pyplot as plt
from torch.optim import Adam
from torch.nn import CrossEntropyLoss
from Trainer import Trainer
from Hyperparameters import Hyperparameters as Hyp
# Set up scheduler breakpoints
step_down = [1, 5, 10, 20]
# Set up trainer
optim_params = Hyp()
optim_params.register('lr')
optim_params.set_value('lr', 1e-4)
for ind in range(2):
for step in step_down:
optim_params = Hyp()
optim_params.register('lr')
optim_params.set_value('lr', 1e-4)
if ind == 1:
optim_params.register('weight_decay')
optim_params.set_value('weight_decay', 3e-2)
tr = Trainer(ResNet50, 'data', device, batch_size=128)
tr.set_hyperparameters(optim_params)
tr.set_criterion(CrossEntropyLoss)
tr.set_optimizer(Adam)
tr.set_scheduler(MultiStepLR)
hidden = Neural_network_layer(8000,Activation_function.sigmoid, Dropout.drop_activation,
frequency = [0], dropout = 0.5, dropout_decay=[0.2], batch_size = batch_size)
#hidden2 = Neural_network_layer(8000,Activation_function.sigmoid, Dropout.drop_activation,
# frequency = 100, dropout = 0.5, dropout_decay=0.2, batch_size = batch_size)
output = Neural_network_layer(10,Activation_function.softmax, Dropout.drop_none, 0,0, batch_size = batch_size, frequency = 0)
layers = [input, hidden,
#hidden2,
output]
H.layers = layers
H.set_learning_parameters(epochs = 100,
learning_rate = 0.1, use_learning_rate_decay = False, learning_rate_decay = 0.01,
use_momentum = True, momentum = 0.5,transient_phase = 1500,
use_momentum_increase = True, momentum_increase = 0.001, use_bias = True)
H.set_standard_regularization_parameters(
use_early_stopping = True,
use_L2 = True, L2_penalty = 0.0001,
use_weight_decay = False, weight_decay = 0.05,weight_decay_decay = 0.01)
H.set_stopping_and_ensemble_parameter(use_ensemble = False, ensemble_count = 15,
safe_weights_threshold = 0.011, epochs_force_reinitilization = 260)
H.set_initial_parameters()
nn = Neural_network(result_dir,data_X, data_y, data_t, data_X_val, data_y_val, data_t_val, X_test,
layers = layers,
import torch.nn as nn
import torch.nn.functional as F
from ResnetModels import ResNet, ResNet50
# Choose device
USE_GPU = True
device = None
if USE_GPU and torch.cuda.is_available():
device = torch.device('cuda')
else:
device = torch.device('cpu')
import numpy as np
import matplotlib.pyplot as plt
from torch.optim import Adam
from torch.nn import CrossEntropyLoss
from Trainer import Trainer
from Hyperparameters import Hyperparameters as Hyp
# Set up trainer
tr = Trainer(ResNet50, 'data', device, batch_size=128)
optim_params = Hyp()
optim_params.register('lr')
optim_params.set_value('lr', 1e-5)
optim_params.register('weight_decay')
optim_params.set_range('weight_decay', -3, -1)
tr.set_hyperparameters(optim_params)
tr.set_criterion(CrossEntropyLoss)
tr.set_optimizer(Adam)
tr.prime_optimizer()
tr.prime_model(pretrained=True)
tr.hyp_opt(epochs=7, iters=25)
#tr.train(epochs=1, save_every=20, update_every=1)
class MetaModel(SerialData):
def __init__(self, hyperparameters: Hyperparameters = Hyperparameters()):
super().__init__()
self.cells: List[MetaCell] = [] # [NORMAL_CELL, REDUCTION_CELL]
self.hyperparameters = hyperparameters
self.model_name = 'evo_' + str(time.time())
self.parent_model_name = ''
self.metrics = Metrics()
self.fitness = 0.
self.keras_model: tf.keras.Model = None
self.keras_model_data: ModelDataHolder = None
def container_name(self):
return self.model_name + '_container'
def populate_with_nasnet_metacells(self):
groups_in_block = 5
ops_in_group = 2
group_inputs = 2
def get_cell():
cell = MetaCell(group_inputs)
cell.groups = [MetaGroup() for _ in range(groups_in_block)]
for i in range(groups_in_block):
cell.groups[i].operations = [
MetaOperation(i + group_inputs)
for _ in range(ops_in_group)
] # +2 because 2 inputs for cell, range(2) because pairwise groups
for j in range(ops_in_group):
cell.groups[i].operations[j].actual_attachment = min(
j, group_inputs - 1)
return cell
def randomize_cell(cell: MetaCell):
for group_ind, group in enumerate(cell.groups):
# do hidden state randomization for all but first groups
if group_ind > 0:
for op in group.operations:
op.actual_attachment = int(np.random.random() *
op.attachment_index)
# do op randomization for all groups
for op in group.operations:
op.operation_type = int(np.random.random() *
OperationType.SEP_1X7_7X1)
normal_cell = get_cell()
reduction_cell = get_cell()
randomize_cell(normal_cell)
randomize_cell(reduction_cell)
self.cells.append(normal_cell)
self.cells.append(reduction_cell)
def mutate(self):
cell_index, group_index, item_index, mutation_type, mutation_subtype = self.select_mutation(
)
self.apply_mutation(cell_index, group_index, item_index, mutation_type,
mutation_subtype)
def select_mutation(self):
cell_index = int(np.random.random() * len(self.cells))
select_block = self.cells[cell_index]
group_index = int(np.random.random() * len(select_block.groups))
select_group = select_block.groups[group_index]
item_index = int(np.random.random() * len(select_group.operations))
mutation_type = np.random.random()
mutation_subtype = np.random.random()
return cell_index, group_index, item_index, mutation_type, mutation_subtype
def apply_mutation(self, cell_index, group_index, item_index,
mutation_type, mutation_subtype):
other_mutation_threshold = (
(1. - self.hyperparameters.parameters['IDENTITY_THRESHOLD']) /
2.) + self.hyperparameters.parameters['IDENTITY_THRESHOLD']
select_block = self.cells[cell_index]
select_group = select_block.groups[group_index]
select_item = select_group.operations[item_index]
mutation_string = f'mutating cell {cell_index}, group {group_index}, item {item_index}: '
if mutation_type < self.hyperparameters.parameters[
'IDENTITY_THRESHOLD']:
# identity mutation
print(mutation_string + 'identity mutation')
return
if self.hyperparameters.parameters[
'IDENTITY_THRESHOLD'] < mutation_type < other_mutation_threshold:
# hidden state mutation = change inputs
# don't try to change the state of the first group since it need to point to the first two inputs of the block
if group_index != 0:
previous_attachment = select_item.actual_attachment
new_attachment = previous_attachment
# ensure that the mutation doesn't result in the same attachment as before
while new_attachment == previous_attachment:
new_attachment = int(
mutation_subtype *
select_item.attachment_index) #TODO: EXCLUSIVE RANDOM
if self.keras_model_data is not None:
self.keras_model_data.hidden_state_mutation(
self.hyperparameters, cell_index, group_index,
item_index, new_attachment, select_item.operation_type)
select_item.actual_attachment = new_attachment
print(
mutation_string +
f'hidden state mutation from {previous_attachment} to {select_item.actual_attachment}'
)
else:
print(mutation_string + f'skipping state mutation for group 0')
else:
# operation mutation
previous_op = select_item.operation_type
select_item.operation_type = int(mutation_subtype *
(OperationType.SEP_1X7_7X1 + 1))
if previous_op != select_item.operation_type and self.keras_model_data is not None:
self.keras_model_data.operation_mutation(
self.hyperparameters, cell_index, group_index, item_index,
select_item.operation_type)
print(
mutation_string +
f'operation type mutation from {previous_op} to {select_item.operation_type}'
)
initial_layer_shape = self.keras_model.layers[0].get_input_shape_at(
0)[1:]
self.keras_model = None
self.build_model(initial_layer_shape, False)
def serialize(self) -> dict:
# return {
# 'blocks': [x.serialize() for x in self.cells],
# 'metrics': self.metrics.serialize(),
# 'hyperparameters': self.hyperparameters.serialize(),
# 'model_name': self.model_name,
# 'parent_model_name': self.parent_model_name
# }
return {
'embedding': self.get_embedding(),
'hyperparameters': self.hyperparameters.serialize(),
'model_name': self.model_name,
'metrics': self.metrics.serialize(),
'parent_model_name': self.parent_model_name
}
def deserialize(self, obj: dict) -> None:
# for block in obj['blocks']:
# item = MetaCell()
# item.deserialize(block)
# self.cells.append(item)
self.populate_from_embedding(obj['embedding'])
self.model_name = obj['model_name']
self.metrics = Metrics()
self.metrics.deserialize(obj['metrics'])
self.hyperparameters = Hyperparameters()
self.hyperparameters.deserialize(obj['hyperparameters'])
if 'parent_model_name' in obj:
self.parent_model_name = obj['parent_model_name']
else:
self.parent_model_name = ''
def build_model(self, input_shape, use_new_weights: bool = True) -> None:
if self.keras_model is None:
print('Creating new keras model')
build_time = time.time()
if self.keras_model_data is None or use_new_weights:
self.keras_model_data = ModelDataHolder(self)
model_input = tf.keras.Input(input_shape)
self.keras_model = self.keras_model_data.build(model_input)
build_time = time.time() - build_time
# optimizer = tf.keras.optimizers.Adam(self.hyperparameters.parameters['MAXIMUM_LEARNING_RATE'])
optimizer = tf.keras.optimizers.SGD(
self.hyperparameters.parameters['MAXIMUM_LEARNING_RATE'])
compile_time = time.time()
self.keras_model.compile(
optimizer=optimizer,
loss=tf.keras.losses.SparseCategoricalCrossentropy(
from_logits=True),
metrics=['accuracy'])
compile_time = time.time() - compile_time
self.metrics.metrics['build_time'] = build_time
self.metrics.metrics['compile_time'] = compile_time
else:
print('reusing previous keras model')
def evaluate(self,
dataset: ImageDataset,
save_interval: int = 0,
save_path: str = None) -> None:
batch_size = self.hyperparameters.parameters['BATCH_SIZE']
min_lr = self.hyperparameters.parameters['MINIMUM_LEARNING_RATE']
max_lr = self.hyperparameters.parameters['MAXIMUM_LEARNING_RATE']
sgdr = SGDR(min_lr, max_lr, batch_size, len(dataset.train_labels),
self.hyperparameters.parameters['SGDR_EPOCHS_PER_RESTART'],
self.hyperparameters.parameters['SGDR_LR_DECAY'],
self.hyperparameters.parameters['SGDR_PERIOD_DECAY'])
completed_iterations = len(self.metrics.metrics['accuracy'])
completed_epochs = completed_iterations * self.hyperparameters.parameters[
'TRAIN_EPOCHS']
if completed_epochs != 0:
sgdr.init_after_epochs(completed_epochs)
callbacks = [self.keras_model_data.drop_path_tracker]
if self.hyperparameters.parameters['USE_SGDR']:
callbacks.append(sgdr)
for iteration in range(
completed_iterations,
self.hyperparameters.parameters['TRAIN_ITERATIONS']):
print(f'Starting training iteration {iteration}')
train_time = time.time()
for epoch_num in range(
int(self.hyperparameters.parameters['TRAIN_EPOCHS'])):
self.keras_model.fit(dataset.train_images,
dataset.train_labels,
shuffle=True,
batch_size=batch_size,
epochs=1,
callbacks=callbacks)
train_time = time.time() - train_time
inference_time = time.time()
evaluated_metrics = self.keras_model.evaluate(
dataset.test_images, dataset.test_labels)
inference_time = time.time() - inference_time
self.metrics.metrics['accuracy'].append(
float(evaluated_metrics[-1]))
self.metrics.metrics['average_train_time'].append(
train_time /
float(self.hyperparameters.parameters['TRAIN_EPOCHS'] *
len(dataset.train_labels)))
self.metrics.metrics['average_inference_time'].append(
inference_time / float(len(dataset.test_images)))
if save_interval != 0 and iteration % save_interval == 0 and save_path is not None:
print('Checkpoint. Saving model...')
self.save_metadata(save_path)
self.save_model(save_path)
def save_metadata(self, dir_path: str = model_save_dir):
dir_name = os.path.join(dir_path, self.model_name)
if not os.path.exists(dir_name):
os.mkdir(dir_name)
SerialData.write_serial_data_to_json(self, dir_name, self.model_name)
def plot_model(self, dir_path):
dir_name = os.path.join(dir_path, self.model_name)
if not os.path.exists(dir_name):
os.mkdir(dir_name)
had_keras_model: bool = self.keras_model is not None
if not had_keras_model:
self.build_model([16, 16, 3])
tf.keras.utils.plot_model(self.keras_model,
os.path.join(dir_name,
self.model_name + '.png'),
expand_nested=True,
show_layer_names=False,
show_shapes=True)
if not had_keras_model:
self.clear_model()
def save_model(self, dir_path: str = model_save_dir):
if self.keras_model is not None:
custom_objects = {
'SeperableConvolutionOperation': SeperableConvolutionOperation,
'AveragePoolingOperation': AveragePoolingOperation,
'MaxPoolingOperation': MaxPoolingOperation,
'DoublySeperableConvoutionOperation':
DoublySeperableConvoutionOperation,
'DimensionalityReductionOperation':
DimensionalityChangeOperation,
'IdentityOperation': IdentityOperation,
'DenseOperation': DenseOperation,
'Relu6Layer': Relu6Layer
}
print(f'saving graph for {self.model_name}')
dir_name = os.path.join(dir_path, self.model_name)
if not os.path.exists(dir_name):
os.mkdir(dir_name)
save_time = time.time()
ModelUtilities.save_keras_model(self.keras_model, dir_name,
self.model_name)
save_time = time.time() - save_time
self.metrics.metrics['save_time'] = save_time
print(
f'finished saving graph for {self.model_name} after {save_time} seconds'
)
def clear_model(self):
if self.keras_model is not None:
del self.keras_model
self.keras_model = None
self.keras_model_data = None
print(f'finished clearing model for {self.model_name}')
def produce_child(self) -> MetaModel:
result: MetaModel = MetaModel(self.hyperparameters)
result.cells = copy.deepcopy(self.cells)
result.keras_model = self.keras_model
result.keras_model_data = self.keras_model_data
self.keras_model = None
self.keras_model_data = None
return result
def load_model(self, dir_path: str = model_save_dir) -> bool:
dir_name = os.path.join(dir_path, self.model_name)
contained_files = os.listdir(dir_name)
contains_keras_model = False
for fl in contained_files:
if len(fl) > 3 and fl[-3:] == '.h5':
contains_keras_model = True
if contains_keras_model:
print(f'loading model for {self.model_name}')
load_time = time.time()
custom_objects = {
'SeperableConvolutionOperation': SeperableConvolutionOperation,
'AveragePoolingOperation': AveragePoolingOperation,
'MaxPoolingOperation': MaxPoolingOperation,
'DoublySeperableConvoutionOperation':
DoublySeperableConvoutionOperation,
'DimensionalityChangeOperation': DimensionalityChangeOperation,
'IdentityReductionOperation': IdentityReductionOperation,
'IdentityOperation': IdentityOperation,
'DenseOperation': DenseOperation,
'Relu6Layer': Relu6Layer,
'DropPathOperation': DropPathOperation
}
self.keras_model = ModelUtilities.load_keras_model(
dir_name, self.model_name, custom_objects)
# print(self.keras_model.summary(line_length=200))
self.keras_model_data = ModelDataHolder(self, self.keras_model)
load_time = time.time() - load_time
print(
f'finished loading model for {self.model_name} in {load_time} seconds'
)
return True
else:
print(f'could not find keras model for {self.model_name}')
return False
@staticmethod
def load(dir_path: str, name: str, load_graph: bool = False) -> MetaModel:
# print(f'loading model, load_graph = {load_graph}')
dir_name = os.path.join(dir_path, name)
if not os.path.exists(dir_name):
print('Model does not exist at specified location')
return MetaModel()
serial_data = SerialData.load_serial_data_from_json(dir_name, name)
result = MetaModel()
result.deserialize(serial_data)
if load_graph:
result.load_model(dir_path)
return result
def generate_graph(self, dir_path: str):
print(f'Generating graph for {self.model_name}')
graph = Digraph(comment='Model Architecture', format='png')
for cell_index, cell in enumerate(self.cells):
graph.node(f'{cell_index}_in', f'Cell Input {cell_index}')
graph.node(f'{cell_index}_0', f'Previous Layer')
graph.node(f'{cell_index}_1', f'Residual')
graph.edge(f'{cell_index}_in', f'{cell_index}_0')
graph.edge(f'{cell_index}_in', f'{cell_index}_1')
for group_index, group in enumerate(cell.groups):
graph.node(f'{cell_index}_{group_index + 2}',
f'Group Concat {cell_index}_{group_index}')
for item_index, item in enumerate(group.operations):
graph.node(
f'{cell_index}_{group_index}_{item_index}',
f'{OperationType.lookup_string(item.operation_type)}')
graph.edge(f'{cell_index}_{item.actual_attachment}',
f'{cell_index}_{group_index}_{item_index}')
graph.edge(f'{cell_index}_{group_index}_{item_index}',
f'{cell_index}_{group_index + 2}')
unused_nodes = cell.get_unused_group_indexes()
graph.node(f'{cell_index}_out', 'Cell Output')
for node in unused_nodes:
graph.edge(f'{cell_index}_{node}', f'{cell_index}_out')
graph.render(os.path.join(dir_path, self.model_name, 'graph.png'))
def get_flops(self, dataset: ImageDataset):
if self.keras_model is None:
return 0
# session = tf.compat.v1.get_default_session()
session = tf.compat.v1.keras.backend.get_session()
with session.as_default():
input_img = tf.ones((1, ) + dataset.images_shape, dtype=tf.float32)
output_image = self.keras_model(input_img)
run_meta = tf.compat.v1.RunMetadata()
_ = session.run(
output_image,
options=tf.compat.v1.RunOptions(
trace_level=tf.compat.v1.RunOptions.FULL_TRACE),
run_metadata=run_meta,
# feed_dict={input_img:np.reshape(dataset.test_images[0], (1,)+dataset.images_shape)}
)
opts = tf.compat.v1.profiler.ProfileOptionBuilder.float_operation()
# opts['output'] = 'none'
flops = tf.compat.v1.profiler.profile(run_meta=run_meta,
cmd='op',
options=opts)
return flops.total_float_ops
# run_meta = tf.compat.v1.RunMetadata()
# opts = tf.compat.v1.profiler.ProfileOptionBuilder.float_operation()
# flops = tf.compat.v1.profiler.profile(tf.compat.v1.keras.backend.get_session().graph, run_meta=run_meta, cmd='op', options=opts)
# return flops.total_float_ops
def get_embedding(self):
embedding = []
embedding.append(self.hyperparameters.parameters['NORMAL_CELL_N'])
embedding.append(self.hyperparameters.parameters['CELL_LAYERS'])
for cell in self.cells:
for group in cell.groups:
for op in group.operations:
embedding.append(op.operation_type)
embedding.append(op.actual_attachment)
return embedding
def populate_from_embedding(self, embedding):
print(f'Populating model from embedding')
num_cells = 2
num_groups_per_cell = 5
num_ops_per_group = 2
num_cell_inputs = 2
dup_embedding = embedding.copy()
self.hyperparameters.parameters['NORMAL_CELL_N'] = dup_embedding[0]
del dup_embedding[0]
self.hyperparameters.parameters['CELL_LAYERS'] = dup_embedding[0]
del dup_embedding[0]
for cell_ind in range(num_cells):
self.cells.append(MetaCell(num_cell_inputs))
for group_ind in range(num_groups_per_cell):
self.cells[cell_ind].groups.append(MetaGroup())
for op_ind in range(num_ops_per_group):
self.cells[cell_ind].groups[group_ind].operations.append(
MetaOperation(num_cell_inputs + group_ind))
ref_op = self.cells[cell_ind].groups[group_ind].operations[
op_ind]
ref_op.operation_type = dup_embedding[0]
ref_op.actual_attachment = dup_embedding[1]
del dup_embedding[0]
del dup_embedding[0]
def get_confusion_matrix(self, dataset):
predictions = self.keras_model.predict(dataset.test_images,
batch_size=32)
predictions = ModelUtilities.softmax(predictions)
predictions = np.argmax(predictions, axis=1)
matrix = tf.math.confusion_matrix(dataset.test_labels,
predictions,
num_classes=10)
matrix_val = None
if not tf.executing_eagerly():
with tf.compat.v1.Session().as_default():
matrix_val = matrix.eval()
else:
matrix_val = matrix.numpy()
return matrix_val
def activation_viewer(self) -> tf.keras.Model:
if self.keras_model is None or self.keras_model_data is None:
return None
parser = ModelParsingHelper()
first_cell_reduce = self.keras_model.get_layer(
parser.get_next_name('concatenate')).get_output_at(0)
# first_cell_reduce = tf.keras.layers.Softmax()(first_cell_reduce)
outputs = [first_cell_reduce]
outputs.extend(self.keras_model.outputs)
output_model = tf.keras.Model(inputs=self.keras_model.inputs,
outputs=outputs)
return output_model
def process_stuff(self):
return [x.process_stuff() for x in self.cells]
@staticmethod
def get_nasnet_embedding() -> List:
return [
5,
3,
OperationType.SEP_3X3,
0, # NORMAL CELL
OperationType.IDENTITY,
0,
OperationType.SEP_3X3,
1,
OperationType.SEP_5X5,
0,
OperationType.AVG_3X3,
0,
OperationType.IDENTITY,
1,
OperationType.AVG_3X3,
1,
OperationType.AVG_3X3,
1,
OperationType.SEP_5X5,
1,
OperationType.SEP_3X3,
1,
OperationType.SEP_7X7,
1, # REDUCTION CELL
OperationType.SEP_5X5,
0,
OperationType.MAX_3X3,
0,
OperationType.SEP_7X7,
1,
OperationType.AVG_3X3,
0,
OperationType.SEP_5X5,
1,
OperationType.MAX_3X3,
0,
OperationType.SEP_3X3,
2,
OperationType.AVG_3X3,
2,
OperationType.IDENTITY,
3
]
@staticmethod
def get_identity_embedding() -> List:
embedding = [5, 3]
embedding.extend([0] * 40)
return embedding
@staticmethod
def get_s1_embedding() -> List:
return [
5, 3, 6, 0, 0, 1, 0, 0, 6, 1, 4, 1, 3, 2, 5, 4, 5, 3, 2, 1, 0, 1,
3, 0, 0, 1, 4, 2, 1, 0, 1, 1, 6, 3, 3, 4, 5, 0, 5, 3, 2, 4
]
@staticmethod
def get_m1_sep7_embedding() -> List:
embedding = [5, 3]
embedding.extend([OperationType.SEP_7X7, 0] * 20)
return embedding
@staticmethod
def get_m1_sep3_embedding() -> List:
embedding = [5, 3]
embedding.extend([OperationType.SEP_3X3, 0] * 20)
return embedding
@staticmethod
def get_m1_sep3_serial_embedding() -> List:
embedding = [5, 3]
for j in range(2):
embedding.extend(
[OperationType.SEP_3X3, 0, OperationType.SEP_3X3, 1])
for i in range(1, 5):
embedding.extend([
OperationType.SEP_3X3, i + 1, OperationType.SEP_3X3, i + 1
])
return embedding
def main():
parser = argparse.ArgumentParser(description='')
parser.add_argument("-ds",
"--dataset",
default="AmazonCat-13K",
type=str,
required=True)
parser.add_argument("-le", "--label_embs", default='elmo', type=str)
parser.add_argument("-gpu", "--device_num", default='1', type=str)
parser.add_argument("-train", "--is_train", default=1, type=int)
parser.add_argument("-ep", "--epochs", default=8, type=int)
parser.add_argument("-ft", "--fine_tune", default=0, type=int)
parser.add_argument("-from", "--ft_from", default=0, type=int)
args = parser.parse_args()
hypes = Hyperparameters(args.dataset)
ds_path = '../datasets/' + args.dataset
device_num = args.device_num
label_embs = args.label_embs
fine_tune = args.fine_tune
epochs = args.epochs
is_train = args.is_train
ft = (args.fine_tune == 1)
ft_from = args.ft_from
trn_X_raw, trn_Y = load_raw(ds_path, 1)
test_X_raw, test_Y = load_raw(ds_path, 0)
bertReg = BertRegressor(hypes,
label_embs,
device_num,
epochs,
ft,
max_seq_len=256)
trn_X_path = ds_path + '/trn_X'
test_X_path = ds_path + '/test_X'
if label_embs == 'elmo' or label_embs == 'en':
label_space = smat.load_npz(ds_path + '/L.elmo.npz')
trn_Y_path = ds_path + '/trn_elmo'
test_Y_path = ds_path + '/test_elmo_Y'
trn_X, trn_Y, trn_Y_nums = load_data(trn_X_raw, trn_Y, trn_X_path,
trn_Y_path, bertReg, label_space,
'elmo')
test_X, test_Y, test_Y_nums = load_data(test_X_raw, test_Y,
test_X_path, test_Y_path,
bertReg, label_space, 'elmo')
else:
print('invalid label embedding type')
exit()
# print('Number of instnaces:', len(trn_X))
# print('Number of labels:', len(trn_Y_nums))
knn_path = ds_path + 'nbrs'
if os.path.isfile(knn_path):
with open(knn_path, 'rb') as f:
nbrs = pkl.load(f)
else:
with open(knn_path, 'wb') as f:
nbrs = NN(n_neighbors=5, algorithm='auto').fit(label_space)
pkl.dump(nbrs, f)
if is_train:
if ft:
print(
'======================Start Fine-tuning======================'
)
model_path = '../save_models/regressor/' + hypes.dataset + '/ep-' + str(
ft_from) + '-elmo/pytorch_model.bin'
bertReg.train(trn_X, trn_Y, trn_Y_nums, label_space, nbrs,
model_path, ft_from)
else:
print('======================Start Training======================')
bertReg.train(trn_X, trn_Y, trn_Y_nums, label_space, nbrs)
output_dir = '../save_models/regressor/' + hypes.dataset + '/ep-' + str(
epochs + ft_from) + '-' + label_embs + '/'
bertReg.save(output_dir)
else:
model_path = '../save_models/regressor/' + hypes.dataset + '/ep-' + str(
ft_from) + '/pytorch_model.bin'
print('======================Start Testing======================')
accs = bertReg.evaluate(test_X, test_Y, test_Y_nums, label_space, nbrs,
model_path)
import read_data
import model
import utils
from Hyperparameters import Hyperparameters
import tensorflow.compat.v1 as tf
import numpy as np
hyperparameters = Hyperparameters()
with tf.Graph().as_default():
# =========================================================================================================
# BUILD MODEL
# =========================================================================================================
train_operation = model.model_architecture(hyperparameters)
# =========================================================================================================
# LOAD DATA
# =========================================================================================================
input_train, train_label, input_test, test_label = read_data.load_data(
hyperparameters.num_points)
scaled_laplacian_train, scaled_laplacian_test = read_data.prepare_data(
input_train, input_test, hyperparameters.num_neighhbors,
hyperparameters.num_points)
# =========================================================================================================
# TRAIN MODEL
# =========================================================================================================
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
def test2():
# More complex integration test
defaults = {
'a': 'a1',
'b': {
'c': 'c1'
},
'd': 'd1',
'e': {
'f': 'f1',
'g': 'g1',
'h': 'h1',
}
}
non_hash = {
'i': 'i1',
'j': {
'k': 'k1',
'l': 'l1',
}
}
inputs = {
'd': 'd2', # Simple override
'e': {
'g': 'g2'
}, # Complex hash override, want to only override 'g',
# but without affecting 'f' or 'h'.
'j': {
'l': 'l2'
} # Complex non-hash override, want to only override
# 'l', but without affecting 'k'
}
correct_integration = {
'a': 'a1',
'b': {
'c': 'c1'
},
'd': 'd2',
'e': {
'f': 'f1',
'g': 'g2',
'h': 'h1',
},
'i': 'i1',
'j': {
'k': 'k1',
'l': 'l2',
}
}
hps = Hyperparameters(inputs, defaults, non_hash)
print_test_result(
hps._integrated_hps, correct_integration,
'test 2a: complex hp integration, partial dict overrides.')
# These will be wrong or throw access errors if implementation is wrong
print_test_result(hps['e:g'], 'g2',
'test 2b: recursive __getattr__') # should be 'g2'
print_test_result(hps['j:k'], 'k1',
'test 2c: recursive __getattr__') # should be 'k1'
# Test partitioning for hashing
# should include d and g but NOT l
print_test_result(hps.hps_to_hash, {
'd': 'd2',
'e': {
'g': 'g2'
}
}, 'test 2d: hp partitioning for hashing')
# Choose device
USE_GPU = True
device = None
if USE_GPU and torch.cuda.is_available():
device = torch.device('cuda')
else:
device = torch.device('cpu')
import numpy as np
import matplotlib.pyplot as plt
from torch.optim import Adam
from torch.nn import CrossEntropyLoss
from torch.optim.lr_scheduler import MultiStepLR, ReduceLROnPlateau
from Trainer import Trainer
from Hyperparameters import Hyperparameters as Hyp
# Set up trainer
tr = Trainer(ResNet50, 'data', device, batch_size=128)
optim_params = Hyp()
optim_params.register('lr')
optim_params.set_value('lr', 1e-4)
tr.set_hyperparameters(optim_params)
tr.set_hyperparameters(optim_params)
tr.set_criterion(CrossEntropyLoss)
tr.set_optimizer(Adam)
tr.set_scheduler(MultiStepLR)
tr.prime_optimizer()
tr.prime_scheduler(milestones=[5], gamma=0.1)
tr.prime_model(pretrained=True)
tr.train(epochs=30, save_every=10)
hist = tr.history
write_history('raw', hist)