def train_coarse(self, training_data, validation_data, fine2coarse):
x_train, y_train = training_data
yc_train = tf.linalg.matmul(y_train, fine2coarse)
x_val, y_val = validation_data
yc_val = tf.linalg.matmul(y_val, fine2coarse)
del y_train, y_val
p = self.training_params
val_thresh = p["validation_loss_threshold"]
logger.debug(f"Creating coarse classifier with shared layers")
self.cc, _ = self.build_cc_fc(verbose=False)
self.fc = None
optim = tf.keras.optimizers.SGD(lr=p['lr_coarse'], nesterov=True, momentum=0.5)
loc = self.save_cc_model()
logger.info('Start Coarse Classification Training')
index = p['initial_epoch']
prev_val_loss = float('inf')
counts_patience = 0
patience = p["patience"]
while index < p['stop']:
tf.keras.backend.clear_session()
self.load_cc_model(loc)
cc = tf.keras.Model(inputs=self.cc.inputs, outputs=self.cc.outputs[1])
cc.compile(optimizer=optim,
loss='categorical_crossentropy',
metrics=['accuracy'])
x_train, yc_train, _ = shuffle_data((x_train, yc_train))
cc_fit = cc.fit(x_train, yc_train,
batch_size=p['batch_size'],
initial_epoch=index,
epochs=index + p["step"],
validation_data=(x_val, yc_val),
callbacks=[self.tbCallback_coarse])
val_loss = cc_fit.history["val_loss"][-1]
loc = self.save_cc_model()
if prev_val_loss - val_loss < val_thresh:
counts_patience += 1
logger.info(f"Counts to early stopping: {counts_patience}/{p['patience']}")
if counts_patience >= patience:
break
elif counts_patience % p["reduce_lr_after_patience_counts"] == 0:
new_val = optim.learning_rate * p["lr_reduction_factor"]
logger.info(f"LR is now: {new_val.numpy()}")
optim.learning_rate.assign(new_val)
self.load_best_cc_model()
loc = self.save_cc_model()
else:
counts_patience = 0
prev_val_loss = val_loss
self.save_best_cc_model()
index += p["step"]
def main(args):
logs_file = get_logs_file()
logs_directory = os.path.dirname(logs_file)
logging.basicConfig(level=logging.DEBUG,
filename=logs_file,
filemode='w')
ch = logging.StreamHandler()
ch.setLevel(args.log_level)
ch.setFormatter(logging.Formatter("%(levelname)s:%(name)s:%(message)s"))
logger = logging.getLogger('')
logger.addHandler(ch)
logger.debug(f'Logs file: {logs_file}')
model_directory = get_model_directory()
logger.debug(f'Models directory: {model_directory}')
data_directory = get_data_directory(args)
logger.debug(f'Data directory: {data_directory}')
results_file = get_results_file()
logger.debug(f'Results file: {results_file}')
logger.info('Getting data')
data = get_data(args.dataset, data_directory)
training_data = data[0]
testing_data = data[1]
fine2coarse = data[2]
n_fine_categories = data[3]
n_coarse_categories = data[4]
logger.info('Building model')
net = models.HDCNN(n_fine_categories,
n_coarse_categories, logs_directory,
model_directory, args)
if args.load_model is not None:
logger.info(f'Loading weights from {args.load_model}')
net.load_models(args.load_model)
if args.train:
logger.info('Entering training')
trdx, trdy, _ = shuffle_data(training_data)
training_data = trdx, trdy
training_data, validation_data = train_test_split(training_data)
net.train_shared_layers(training_data, validation_data)
net.save_all_models(model_directory)
net.sync_parameters()
net.train_coarse_classifier(
training_data, validation_data, fine2coarse)
net.save_all_models(model_directory)
net.train_fine_classifiers(
training_data, validation_data, fine2coarse)
net.save_all_models(model_directory)
if args.test:
logger.info('Entering testing')
net.predict(testing_data, fine2coarse, args.results_file)
def train(self, training_data, validation_data):
x_train, y_train = training_data
x_val, y_val = validation_data
p = self.training_params
val_thresh = p["validation_loss_threshold"]
logger.info('Start Full Classification training')
index = p['initial_epoch']
tf.keras.backend.clear_session()
self.full_model = self.build_model(verbose=False)
optim = tf.keras.optimizers.SGD(lr=p['lr'],
nesterov=True,
momentum=0.5)
self.full_model.compile(optimizer=optim,
loss='categorical_crossentropy',
metrics=['accuracy'])
loc = self.save_full_model()
tf.keras.backend.clear_session()
prev_val_loss = float('inf')
counts_patience = 0
patience = p["patience"]
while index < p['stop']:
tf.keras.backend.clear_session()
self.load_full_model(loc)
x_train, y_train, _ = shuffle_data((x_train, y_train))
full_fit = self.full_model.fit(x_train,
y_train,
batch_size=p['batch_size'],
initial_epoch=index,
epochs=index + p["step"],
validation_data=(x_val, y_val),
callbacks=[self.tbCallback_full])
val_loss = full_fit.history["val_loss"][-1]
loc = self.save_full_model()
if prev_val_loss - val_loss < val_thresh:
counts_patience += 1
logger.info(
f"Counts to early stopping: {counts_patience}/{p['patience']}"
)
if counts_patience >= patience:
break
elif counts_patience % p[
"reduce_lr_after_patience_counts"] == 0:
new_val = optim.learning_rate * p["lr_reduction_factor"]
logger.info(f"LR is now: {new_val.numpy()}")
optim.learning_rate.assign(new_val)
self.load_best_full_model()
self.save_full_model()
else:
counts_patience = 0
prev_val_loss = val_loss
self.save_best_full_model()
index += p["step"]
def main(args):
logs_file = get_logs_file(args.name)
logs_directory = os.path.dirname(logs_file)
logging.basicConfig(
level=logging.DEBUG,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
filename=logs_file,
filemode='w')
ch = logging.StreamHandler()
ch.setLevel(args.log_level)
ch.setFormatter(
logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
logger = logging.getLogger('')
logger.addHandler(ch)
logger.debug(f'Logs file: {logs_file}')
model_directory = get_model_directory(args)
logger.debug(f'Models directory: {model_directory}')
data_directory = get_data_directory(args)
logger.debug(f'Data directory: {data_directory}')
results_file = get_results_file(args)
logger.debug(f'Results file: {results_file}')
logger.info('Getting data')
data = get_data(args.dataset, data_directory)
training_data = data[0]
testing_data = data[1]
validation_data = data[2]
fine2coarse = data[3]
n_fine_categories = data[4]
n_coarse_categories = data[5]
input_shape = training_data[0][0].shape
logger.info('Building model')
net = models.VanillaResNet(n_fine_categories=n_fine_categories,
n_coarse_categories=n_coarse_categories,
input_shape=input_shape,
logs_directory=logs_directory,
model_directory=model_directory,
args=args)
if args.train:
logger.info('Entering training')
trdx, trdy, _ = shuffle_data(training_data)
training_data = trdx, trdy
net.train(training_data, validation_data)
if args.test:
logger.info('Entering testing')
net.predict_fine(testing_data, results_file,
fine2coarse) # args.results)
def get_data(dataset, data_directory):
if dataset == 'cifar100':
logging.info('Getting CIFAR-100 dataset')
tr, te, fine2coarse, n_fine, n_coarse = datasets.get_cifar100(
data_directory)
tr_x, tr_y, _ = shuffle_data(tr, random_state=0)
tr = tr_x, tr_y
tr, val = train_test_split(tr)
logging.debug(
f'Training set: x_dims={tr[0].shape}, y_dims={tr[1].shape}')
logging.debug(
f'Testing set: x_dims={te[0].shape}, y_dims={te[1].shape}')
return tr, te, val, fine2coarse, n_fine, n_coarse
def train(self, training_data, validation_data):
x_val, y_val = validation_data
p = self.training_params
optim = tf.keras.optimizers.SGD(lr=p['lr'])
index = p['initial_epoch']
prev_val_loss = float('inf')
counts_patience = 0
self.save_model(self.model_directory + "/vanilla_tmp.h5", self.full_classifier)
while index < p['stop']:
tf.keras.backend.clear_session()
self.full_classifier = self.load_model(self.model_directory + "/vanilla_tmp.h5")
self.full_classifier.compile(optimizer=optim,
loss='categorical_crossentropy',
metrics=['accuracy'])
# logger.info('Training coarse stage')
x_train, y_train, _ = shuffle_data(training_data)
fc = self.full_classifier.fit(x_train, y_train,
batch_size=p['batch_size'],
initial_epoch=index,
epochs=index + p['step'],
validation_data=(x_val, y_val),
callbacks=[self.tbCallback])
val_loss = fc.history['val_loss'][0]
self.save_model(self.model_directory + "/vanilla_tmp.h5", self.full_classifier)
if prev_val_loss - val_loss < p['val_thresh']:
counts_patience += 1
logger.info(f"Counts to early stopping: {counts_patience}/{p['patience']}")
if counts_patience >= p['patience']:
break
elif counts_patience % p["reduce_lr_after_patience_counts"] == 0:
self.full_classifier = self.load_model(self.model_directory + "/vanilla.h5")
new_val = optim.learning_rate * p["lr_reduction_factor"]
logger.info(f"LR is now: {new_val.numpy()}")
optim.learning_rate.assign(new_val)
else:
counts_patience = 0
prev_val_loss = val_loss
self.save_model(self.model_directory + "/vanilla.h5", self.full_classifier)
index += p['step']
def train_both(self, training_data, validation_data, fine2coarse):
x_train, y_train = training_data
x_val, y_val = validation_data
yc_train = tf.linalg.matmul(y_train, fine2coarse)
yc_val = tf.linalg.matmul(y_val, fine2coarse)
p = self.training_params
val_thresh = p["validation_loss_threshold"]
logger.info('Start Full Classification training')
index = p['initial_epoch']
tf.keras.backend.clear_session()
self.load_best_cc_model()
self.load_best_fc_model()
loc_cc = self.save_cc_model()
loc_fc = self.save_fc_model()
tf.keras.backend.clear_session()
optim = tf.keras.optimizers.SGD(lr=p['lr_full'],
nesterov=True,
momentum=0.5)
prev_val_loss = float('inf')
counts_patience = 0
patience = p["patience"]
while index < p['stop']:
tf.keras.backend.clear_session()
self.load_cc_model(loc_cc)
self.load_fc_model(loc_fc)
self.build_full_model()
for l in self.cc.layers:
l.trainable = True
for l in self.fc.layers:
l.trainable = True
self.full_model.compile(optimizer=optim,
loss='categorical_crossentropy',
metrics=['accuracy'])
x_train, y_train, inds = shuffle_data((x_train, y_train))
yc_train = tf.gather(yc_train, inds)
full_fit = self.full_model.fit(x_train, [y_train, yc_train],
batch_size=p['batch_size'],
initial_epoch=index,
epochs=index + p["step_full"],
validation_data=(x_val,
[y_val, yc_val]),
callbacks=[self.tbCallback_full])
val_loss = full_fit.history["val_loss"][-1]
loc_cc = self.save_cc_model()
loc_fc = self.save_fc_model()
if prev_val_loss - val_loss < val_thresh:
counts_patience += 1
logger.info(
f"Counts to early stopping: {counts_patience}/{p['patience']}"
)
if counts_patience >= patience:
break
elif counts_patience % p[
"reduce_lr_after_patience_counts"] == 0:
new_val = optim.learning_rate * p["lr_reduction_factor"]
logger.info(f"LR is now: {new_val.numpy()}")
optim.learning_rate.assign(new_val)
self.load_best_fc_both_model()
self.save_fc_model()
self.load_best_cc_both_model()
self.save_cc_model()
else:
counts_patience = 0
prev_val_loss = val_loss
self.save_best_cc_both_model()
self.save_best_fc_both_model()
index += p["step_full"]
