def __init__(self, X_val, Y_val, check_point, flatten):
Callback.__init__(self)
self.X_val = X_val
self.Y_val = Y_val
self.ssim_max = 0
self.check_point = check_point
self.flatten = flatten
def __init__(self, dataset_train, exp_folder, relabel_method, prop):
self.exp_folder = exp_folder
self.method = relabel_method
self.prop = prop
self.trigger_modulo = cfg.RELABEL.TRIGGER_EPOCH
self.dataset_train = dataset_train
Callback.__init__(self)
def __init__(self, X_data, y_label, model):
"""
:param X_data: X_val
:param y_true: y_val
:param model:
"""
Callback.__init__(self)
self.X_data = X_data
self.y_label = y_label
self.model = model
self.spearmanr_list = []
def __init__(self, num_epochs, progressbar, label):
Callback.__init__(self)
self.num_epochs = num_epochs
self.progressbar = progressbar
self.label = label
self.progress = None
self.increment = None
self.i = None
self.x = None
self.losses = None
self.val_losses = None
self.logs = None
def train(weights_path, epochs, batch_size, initial_epoch, kl_start_epoch,
kl_alpha_increase_per_epoch):
"""Trains a model."""
print('loading data...')
# Loads or creates training data.
input_shape, train, valid, train_targets, valid_targets = get_train_data()
print('getting model...')
# Loads or creates model.
model, checkpoint_path, kl_alpha = get_model(input_shape,
scale_factor=len(train) /
batch_size,
weights_path=weights_path)
# Sets callbacks.
checkpointer = ModelCheckpoint(checkpoint_path,
verbose=1,
save_weights_only=True,
save_best_only=True)
scheduler = LearningRateScheduler(schedule)
annealer = Callback() if kl_alpha is None else AnnealingCallback(
kl_alpha, kl_start_epoch, kl_alpha_increase_per_epoch)
print('fitting model...')
# Trains model.
model.fit(train,
train_targets,
batch_size,
epochs,
initial_epoch=initial_epoch,
callbacks=[checkpointer, scheduler, annealer],
validation_data=(valid, valid_targets))
def __init__(self, seq, batch_size: int, seqlen: int, n_feat: int):
Callback.__init__(self)
self.seq = seq
self.batch_size = batch_size
self.seqlen = seqlen
self.n_feat = n_feat
self.X = np.zeros(
(self.batch_size * len(self.seq), self.seqlen, self.n_feat))
self.y = np.zeros((self.batch_size * len(self.seq), 1))
self.d = np.ones((self.batch_size * len(self.seq), 1),
dtype=np.float32)
idx = 0
for bidx in range(0, len(seq)):
bin, bout, _ = seq.__getitem__(bidx)
self.X[idx:idx + batch_size] = bin
self.y[idx:idx + batch_size] = bout
idx += batch_size
def __init__(self, save_all_models=False):
Callback.__init__(self)
self.save_all_models = save_all_models
get_custom_objects()['PermanentDropout'] = PermanentDropout
def __init__(self, print_fcn=print):
Callback.__init__(self)
self.print_fcn = print_fcn
def run(_run):
# Load configs, if parameters are unspecified, fill in a default
config = _run.config
run = config.get('fit_params')
model_params = config.get('model_params')
data_params = config.get('data_params')
batch_size = data_params.get('batch_size')
stretch_colorspace = data_params.get('stretch_colorspace')
augmentations = data_params.get('augmentations')
bridge_separate = data_params.get(
'bridge_separate') # bridges as separate labels?
buffer_size = data_params.get(
'buffer_size') # the buffer sizes for shuffling
use_sampling = data_params.get('use_sampling')
class_target_prob = 1 / model_params.get('num_classes')
print("[!] list of parameter configurations")
pprint(config)
# parameter assertion test
if model_params.get('num_classes') not in [2, 3]:
raise ValueError("num classes must be in 2,3")
elif (model_params.get('num_classes') == 2 and bridge_separate) or \
(model_params.get('num_classes') ==3 and bridge_separate == False):
raise ValueError("this configuration is not possible")
else:
print("num classes and bridge separate match")
# Load data and define generators ------------------------------------------
print("[!] loading datasets \n")
x_train, y_train, x_val, y_val, x_test, y_test, probs = load_data(
bridge_separate)
# get a rough estimate: there are 100 files per TFRecord
# except for one TFRecord per item, so this estimate might not be 100% correct
num_training = len(x_train) * 100
# TF parsing functions
print("[!] Creating dataset iterators \n")
# Load the dataset iterators
train_dataset = create_training_dataset(x_train, batch_size,
bridge_separate, buffer_size,
stretch_colorspace,
use_sampling, probs,
class_target_prob,
augmentations, **model_params)
val_dataset = validate(x_val, batch_size, bridge_separate,
stretch_colorspace, **model_params)
test_dataset = validate(x_test, batch_size, bridge_separate,
stretch_colorspace, **model_params)
# Model definitions --------------------------------------------------------
print("[!] compiling model and adding callbacks \n")
# function for building the model
model_func = model_dict[run.get('model')]
# invoke the user function
model = model_func(**model_params)
model.summary()
# compile the model with catcrossentropy: one hot encoded labels!!
model.compile(optimizer=tf.keras.optimizers.Adam(run.get('lr')),
loss='categorical_crossentropy',
metrics=['accuracy'])
# Model callbacks ----------------------------------------------------------
# ReduceLRonPlateau
if run.get('reduce_lr_on_plateau'):
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
patience=4,
min_lr=10e-5,
verbose=1)
else:
reduce_lr = Callback()
# Model checkpoints
now = datetime.datetime.now()
date_string = "_".join(
map(str, (now.year, now.month, now.day, now.hour, now.minute,
now.second)))
modelcheckpoint_name = "checkpoints/model-{}-{}-{}-{}.hdf5".format(
run.get('model'), date_string, model_params.get('num_classes'),
len(model_params.get('channels')))
# if reproduce_result:
# modelcheckpoint_name = "../checkpoints/model-{}.hdf5".format(reproduce_result)
modelcheckpoint = ModelCheckpoint(modelcheckpoint_name,
monitor='val_loss',
verbose=1,
save_best_only=True,
save_weights_only=True)
# Model early stopping
earlystopping = EarlyStopping(monitor='val_loss', patience=10)
# Model Training and evaluation --------------------------------------------
print("[!] fitting model \n")
model.fit(train_dataset.repeat(),
epochs=run.get('epochs'),
steps_per_epoch=num_training / batch_size,
validation_data=val_dataset,
validation_steps=None,
shuffle=True,
verbose=1,
callbacks=[
LogMetrics(), modelcheckpoint, earlystopping, reduce_lr
])
# Model evaluation
print("[!] predicting test set")
# load optimal weights
model.load_weights(modelcheckpoint_name)
# evaluate works like a charm
results = model.evaluate(test_dataset)
print("results are", results)
print("[!] predicting confusion matrix")
preds = model.predict(test_dataset)
labels = [label for img, label in test_dataset]
labels = [
np.argmax(item.numpy()) for sublist in labels for item in sublist
]
labels = np.array(labels)
confusion_matrix = tf.math.confusion_matrix(labels,
np.argmax(preds, axis=1))
print(confusion_matrix)
_run.log_scalar("test_loss", float(results[0]))
_run.log_scalar("test_acc", float(results[1]))
_run.log_
def run(_run):
# Load configs, if parameters are unspecified, fill in a default
config = _run.config
run = config.get('fit_params')
model_params = config.get('model_params')
data_params = config.get('data_params')
batch_size = data_params.get('batch_size')
augmentations = data_params.get('augmentations')
buffer_size = data_params.get('buffer_size') # the buffer sizes for shuffling
use_sampling = data_params.get('use_sampling')
class_target_prob = 1 / model_params.get('num_classes')
print("[!] list of parameter configurations")
pprint(config)
# Load data and define generators ------------------------------------------
print("[!] loading datasets \n")
x_train, x_val, x_test, probs = load_data()
# get a rough estimate: there are 100 files per TFRecord
# except for one TFRecord per item, so this estimate might not be 100% correct
num_training = len(x_train) * 100
# TF parsing functions
print("[!] Creating dataset iterators \n")
# Load the dataset iterators
train_dataset = create_training_dataset(x_train, batch_size, buffer_size, augmentations,
use_sampling, probs, class_target_prob,
**model_params)
val_dataset = validate(x_val, batch_size, **model_params)
test_dataset = validate(x_test, batch_size, **model_params)
# we need the actual labels from the TFRecords, but they take INCREDIBLY long to parse
# parse through them once, and create a csv file with a list of all the labels
# note: the tf parsing requires that there is no randomness (shuffling) in the validation/test labels
if not os.path.exists('../datasets/data/valid/val_labels.csv'):
print(os.path.exists('../datasets/data/valid/val_labels.csv'))
print("[!] creating validation label file in ../datasets/data/valid/val_labels.csv")
create_label_csv(val_dataset,'../datasets/data/valid/val_labels.csv')
else:
print("[!] validation labels csv exist")
if not os.path.exists('../datasets/data/test/test_labels.csv'):
print("[!] creating test label file in ../datasets/data/test/test_labels.csv")
create_label_csv(test_dataset,'../datasets/data/test/test_labels.csv')
else:
print("[!] test labels csv exist")
# load the file with validation labels
# getting labels from a TFRecords with lots of other data is horribly slow...
print("[!] Loading validation labels for callbacks")
val_labels = pd.read_csv('../datasets/data/valid/val_labels.csv')
val_labels = np.squeeze(val_labels.to_numpy())
# Model definitions --------------------------------------------------------
print("[!] compiling model and adding callbacks \n")
# function for building the model
model_func = model_dict[run.get('model')]
# invoke the user function
model = model_func(**model_params)
model.summary()
# compile the model with catcrossentropy: one hot encoded labels!!
model.compile(optimizer= tf.keras.optimizers.Adam(run.get('lr')),
loss= 'categorical_crossentropy',
metrics=['accuracy'])
# Model callbacks ----------------------------------------------------------
# ReduceLRonPlateau
if run.get('reduce_lr_on_plateau'):
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3, min_lr=10e-7, verbose=1)
else:
reduce_lr = Callback()
# Model checkpoints
now = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
aug_string = 'aug' if augmentations==True else 'noaug'
modelcheckpoint_name= lambda x: "checkpoints/model-{}-{}-{}-{}-{}.hdf5".format(run.get('model'),
x,
aug_string,
'ch_' + str(len(model_params.get('channels'))),
now)
modelcheckpoint = ModelCheckpoint(modelcheckpoint_name('best_loss'),
monitor = 'val_loss',
verbose=1,
save_best_only=True,
save_weights_only=True)
# Model early stopping
earlystopping = EarlyStopping(monitor='val_loss', patience=10)
# tensorboard and metric callbacks
log_dir = "logs/fit/{}-{}-{}-{}".format(run.get('model'), aug_string, 'ch_' + str(len(model_params.get('channels'))), now)
file_writer = tfsum.create_file_writer(log_dir)
tensorboard_cb = tf.keras.callbacks.TensorBoard(log_dir=log_dir,
histogram_freq=1,
profile_batch=0)
f1_metric = Metrics(val_dataset,
val_labels,
save_best=True,
save_name= modelcheckpoint_name('best_f1'),
writer=file_writer)
# Model Training and evaluation --------------------------------------------
print("[!] fitting model \n")
model.fit(
train_dataset.repeat(),
epochs=run.get('epochs'),
steps_per_epoch= int(num_training / batch_size),
validation_data=val_dataset,
validation_steps = None,
shuffle=True,
verbose= 1,
callbacks = [tensorboard_cb, f1_metric, LogMetrics(), modelcheckpoint, earlystopping, reduce_lr, MemoryCallback()]
)
print("[!] done running, terminating program")
'''
train_horse_names = os.listdir(train_horse_dir)
train_human_names = os.listdir(train_human_dir)
print(len(train_human_names))
print(len(train_horse_names))
# fallback class to stop at perticular accuracy
class Callback(tf.keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs={}):
if(logs.get('accuracy')>0.9):
print("\nReached 90% accuracy so cancelling training!")
self.model.stop_training = True
myCallback = Callback()
# building model for classification
model = tf.keras.Sequential([
# Convolution of images for only effective features
# first convolution
tf.keras.layers.Conv2D(16,(3,3),activation='relu',input_shape=(300,300,3)),
tf.keras.layers.MaxPooling2D(2,2),
# The second convolution
tf.keras.layers.Conv2D(32, (3,3), activation='relu'),
tf.keras.layers.MaxPooling2D(2,2),
# The third convolution
def __init__(self):
Callback.__init__(self)
def __init__(self, val_x, val_y, batch_size=500):
Callback.__init__(self)
self.val_x = val_x
self.val_y = val_y
self.batch_size = batch_size
def __init__(self, name: str, mod: int):
Callback.__init__(self)
self._ctr = 0
self._name = name
self._n_saved = 0
self._mod = mod
def on_epoch_end(self, epoch, logs=None):
if epoch in self.epochs:
generate_sequence(self.model)
def generate_sequence(model):
start = numpy.random.randint(0, len(dataX) - 1)
pattern = dataX[start]
print("Seed:")
print("\"", ''.join([int_to_char[value] for value in pattern]), "\"")
for i in range(1000):
x = numpy.reshape(pattern, (1, len(pattern), 1))
x = x / float(n_vocab)
prediction = model.predict(x, verbose=0)
index = numpy.argmax(prediction)
result = int_to_char[index]
print(result, end='')
pattern.append(index)
pattern = pattern[1:len(pattern)]
model = Sequential()
model.add(LSTM(256, input_shape=(X.shape[1], X.shape[2])))
model.add(Dropout(0.2))
model.add(Dense(y.shape[1], activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam')
filepath="weights-improvement-{epoch:02d}-{loss:.4f}.hdf5"
checkpoint = ModelCheckpoint(filepath, monitor='loss', verbose=1, save_best_only=True, mode='min')
callbacks_list = [checkpoint, Callback([1, 9, 18])]
model.fit(X, y, epochs=20, batch_size=128, callbacks=callbacks_list)
