def _test_reduce_to_keras_loss(self, y_true, y_pred, from_logits: bool):
"""Focal loss with gamma=0 should be the same as cross-entropy."""
keras_loss = tf.keras.losses.SparseCategoricalCrossentropy(
from_logits=from_logits)
focal_loss = SparseCategoricalFocalLoss(gamma=0,
from_logits=from_logits)
self.assertAllClose(keras_loss(y_true, y_pred),
focal_loss(y_true, y_pred))
def build_model(self, input_shape, nb_classes):
input_layer = keras.layers.Input(input_shape)
masked_layer = keras.layers.Masking(mask_value=-1000)(input_layer)
x = masked_layer
input_res = masked_layer
mask = masked_layer[:, :, 0]
for d in range(self.depth):
x = self._inception_module(x, mask)
if self.use_residual and d % 3 == 2:
input_res = keras.layers.Lambda((lambda x: x))(input_res,
mask=mask)
x = self._shortcut_layer(input_res, x)
input_res = x
# x = keras.layers.Dropout(0.2)(x)
gap_layer = keras.layers.GlobalAveragePooling1D()(x, mask=mask)
output_layer = keras.layers.Dense(self.nb_filters)(gap_layer)
output_layer = keras.layers.LeakyReLU()(output_layer)
output_layer = keras.layers.Dense(nb_classes,
activation='softmax')(output_layer)
# model = keras.models.Model(inputs=input_layer, outputs=output_layer)
model = keras.models.Model(inputs=input_layer, outputs=output_layer)
# model.compile(loss='categorical_crossentropy', optimizer=keras.optimizers.Adam(self.lr),
# metrics=['accuracy'])
loss = SparseCategoricalFocalLoss(gamma=3)
model.compile(loss=loss,
optimizer=keras.optimizers.Adam(self.lr),
metrics=['accuracy'])
reduce_lr = keras.callbacks.ReduceLROnPlateau(monitor='loss',
actor=0.5,
patience=50,
min_lr=0.0001)
file_path = self.output_directory + 'best_model.hdf5'
model_checkpoint = keras.callbacks.ModelCheckpoint(
filepath=file_path,
monitor='val_accuracy',
save_best_only=True,
mode='max')
stop_early = keras.callbacks.EarlyStopping(monitor='val_loss',
restore_best_weights=True,
patience=300)
schedule = StepDecay(initAlpha=self.lr, factor=0.85, dropEvery=20)
lr_decay = keras.callbacks.LearningRateScheduler(schedule)
self.callbacks = [reduce_lr, model_checkpoint, stop_early, lr_decay]
return model
def test_get_config(self, gamma, from_logits):
"""Check the get_config() method."""
loss1 = SparseCategoricalFocalLoss(gamma=gamma,
from_logits=from_logits,
name='focal_loss')
config1 = loss1.get_config()
loss2 = SparseCategoricalFocalLoss(**config1)
config2 = loss2.get_config()
self.assertEqual(config1, config2)
def test_with_higher_rank_inputs(self):
"""Addresses https://github.com/artemmavrin/focal-loss/issues/5"""
def build_model():
return tf.keras.Sequential([
tf.keras.layers.Input((100, 10)),
tf.keras.layers.GRU(13, return_sequences=True),
tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(13)),
])
x = np.zeros((20, 100, 10))
y = np.ones((20, 100, 1))
model = build_model()
loss = SparseCategoricalFocalLoss(gamma=2)
model.compile(loss=loss, optimizer='adam')
model.fit(x, y)
def get_dummy_sparse_multiclass_classifier(n_features, n_classes, gamma,
from_logits):
activation = None if from_logits else 'softmax'
# Just a linear classifier (without bias term)
model = tf.keras.Sequential(layers=[
tf.keras.layers.Input(shape=n_features),
tf.keras.layers.Dense(
units=n_classes, use_bias=False, activation=activation),
])
model.compile(
optimizer='sgd',
loss=SparseCategoricalFocalLoss(gamma=gamma, from_logits=from_logits),
metrics=['accuracy'],
)
return model
interpolation='bilinear')(layer)
model_output = tf.keras.layers.Conv2D(num_classes,
kernel_size=(1, 1),
padding='same')(layer)
return tf.keras.Model(inputs=model_input, outputs=model_output)
return DeeplabV3Plus(num_classes)
model = get_model()
model.summary()
# tf.keras.utils.plot_model(model, to_file="model.png",show_shapes=True, dpi=64)
# loss = tf.losses.SparseCategoricalCrossentropy(from_logits=True)
loss = SparseCategoricalFocalLoss(from_logits=True, gamma=2)
step_per_epoch = len(train_images_folder) // batch_size
model.compile(optimizer=tf.keras.optimizers.SGD(learning_rate=0.01,
momentum=0.9,
nesterov=True),
loss=loss,
metrics=['accuracy'])
def scheduler(epoch, lr):
if epoch < 4:
return lr
else:
return lr * tf.math.exp(-0.1)
'--generate_images',
help=
'saves representative image of mask in addition to the mask prediction',
action="store_true")
args = parser.parse_args()
assert os.path.isfile(args.weights), "Could not find weights file!"
assert os.path.isdir(args.images), "Could not find images directory!"
unet = Unet(DESIRED_SIZE,
DESIRED_SIZE,
nclasses=NUM_CLASSES,
filters=UNET_FILTERS)
unet.load_weights(args.weights)
unet.compile(optimizer='adam',
loss=SparseCategoricalFocalLoss(gamma=2),
metrics=['accuracy'])
image_paths = glob.glob(args.images + '/*.jpg')
images = [
x[0] for x in [
get_resized_image_and_mask_label(cv2.imread(p), NEGATIVE)
for p in image_paths
]
]
mpreds = unet.predict(np.array(images, dtype=np.uint8))
preds = maskpred(mpreds)
cmap = copy.copy(plt.cm.get_cmap("gist_rainbow", 20))
cmap.set_under(color="black")
from_logits = st.text_input('From Logits (True or False):')
if from_logits != "":
from_logits = bool(from_logits)
else:
from_logits = False
focal_kwargs = st.text_input(
'Keyword Arguments (e.g. {"name":"focal_loss",...}):')
if focal_kwargs != "":
focal_kwargs = eval(focal_kwargs)
else:
focal_kwargs = {}
loss_fn = SparseCategoricalFocalLoss(gamma=gamma,
alpha=alpha,
from_logits=from_logits,
**focal_kwargs)
# Restricted dynamic sampling parameters
st.write("""
## Restricted Dynamic Sampling Parameters
""")
dynamic_sampling_start_epoch = st.text_input(
'Dynamic Sampling Start Epoch (e.g. 10):')
if dynamic_sampling_start_epoch != "":
dynamic_sampling_start_epoch = int(dynamic_sampling_start_epoch)
else:
dynamic_sampling_start_epoch = 0
data_dimensions = st.text_input(
'Data Dimensions (Dimensions of one sample in dataset):')
def restore_checkpoint():
if checkpoint_manager.latest_checkpoint:
checkpoint.restore(checkpoint_manager.latest_checkpoint)
print("Model restored at epoch {}".format(checkpoint.epoch.numpy()))
with strategy.scope():
model = get_model()
optimizer = tf.keras.optimizers.SGD(learning_rate=0.01, momentum=0.9, nesterov=True)
train_loss_metric = tf.keras.metrics.Mean('train_loss', dtype=tf.float32)
train_acc_metric = tf.keras.metrics.CategoricalAccuracy('train_accuracy')
test_loss_metric = tf.keras.metrics.Mean('test_loss', dtype=tf.float32)
test_acc_metric = tf.keras.metrics.CategoricalAccuracy('test_accuracy')
loss_object = SparseCategoricalFocalLoss(from_logits=True, gamma=2, reduction=tf.keras.losses.Reduction.NONE)
# loss_object = tf.keras.losses.SparseCategoricalCrossentropy(
# from_logits=True, reduction=tf.keras.losses.Reduction.NONE)
global_batch_size = batch_size * strategy.num_replicas_in_sync
# task_type = strategy.cluster_resolver.task_type
# task_id = strategy.cluster_resolver.task_id
print('task_type:{} task_id:{} num_replicas_in_sync:{}'.format(task_type, task_id,strategy.num_replicas_in_sync))
# task_type, task_id = 'chief', 0
is_chief = _is_chief(task_type, task_id)
checkpoint = tf.train.Checkpoint(epoch=tf.Variable(1),
val_loss=tf.Variable(np.inf), model=model, optimizer=optimizer)
write_model_path = write_filepath('slurm_saved_models', task_type, task_id)
write_checkpoint_dir = write_filepath('slurm_checkpoints', task_type, task_id)
train_dataset = dataset.get_dataset(False, True)
# Define evaluation dataset
eval_dataset = dataset.get_dataset(False, False)
# We need the batches number in evaluation dataset, so here is:
# (This will be executed in eager mode)
n_eval_batches = dataset.n_batches_in_dataset(eval_dataset)
# Create model
model = create_model(False)
model.compile(
optimizer=tf.keras.optimizers.Adam(learning_rate=0.001),
#loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
loss=SparseCategoricalFocalLoss(gamma=3, from_logits=True),
metrics=['accuracy'])
model.summary()
# Tensorboard
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=settings.get_model_path(False, 'logs'))
# Save checkpoints
checkpoint_file_format = settings.get_model_path(False, Prediction.CHECKPOINTS_DIR) + '/cp-{epoch:04d}.ckpt'
cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_file_format,
save_weights_only=True,
verbose=1)
# Do real evaluation callback:
# TODO: Performance of this could be improved A LOT
def train_nn(model,
predict,
X_train,
Y_train,
X_test,
Y_test,
fold=0,
epochs=50,
batch_size=512,
results_path='./',
model_name='FCNN'):
assert predict in ('pT', '1/pT', 'pT_classes')
test_index = list(X_test.index)
X_train = X_train.reset_index(drop=True)
Y_train = Y_train.reset_index(drop=True)
X_test = X_test.reset_index(drop=True)
Y_test = Y_test.reset_index(drop=True)
if model_name == 'CNN':
X_train = X_train.to_numpy().reshape((-1, 4, 4, 1))
X_test = X_test.to_numpy().reshape((-1, 4, 4, 1))
checkpoint = ModelCheckpoint("model.h5",
monitor='val_loss',
verbose=0,
save_best_only=True,
mode='min')
early_stop = EarlyStopping(monitor='val_loss', patience=3, verbose=0)
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=0,
verbose=0)
if predict == '1/pT':
model.compile(optimizer='adam', loss='mse')
if predict == 'pT_classes':
model.compile(optimizer='adam',
loss=SparseCategoricalFocalLoss(gamma=2))
if predict == 'pT':
model.compile(optimizer='adam', loss=pTLossTF)
history = model.fit(x=X_train,
y=Y_train,
batch_size=batch_size,
epochs=epochs,
verbose=0,
validation_split=0.1,
callbacks=[checkpoint, early_stop, reduce_lr])
model.load_weights("model.h5")
P = model.predict(X_test)
OOF_preds = pd.DataFrame()
OOF_preds['row'] = test_index
OOF_preds['true_value'] = Y_test[Y_test.columns[0]].to_list()
if predict in ('pT', '1/pT'):
OOF_preds['preds'] = P.reshape((len(X_test)))
else:
OOF_preds['0-10'] = P[:, 0].reshape((len(X_test)))
OOF_preds['10-30'] = P[:, 1].reshape((len(X_test)))
OOF_preds['30-100'] = P[:, 2].reshape((len(X_test)))
OOF_preds['100-inf'] = P[:, 3].reshape((len(X_test)))
OOF_preds.to_csv(os.path.join(results_path,
'OOF_preds_' + str(fold) + '.csv'),
index=False)
return history