def test_misc(self): # tests of non-main features to improve coverage
for optimizer_name in ['AdamW', 'NadamW', 'SGDW']:
cprint("<< TESTING {} OPTIMIZER >>".format(optimizer_name), 'blue')
reset_seeds()
optimizer_kw = {
'total_iterations': 0,
'decay': 1e-3,
'amsgrad': optimizer_name == 'AdamW',
'nesterov': optimizer_name == 'SGDW'
}
num_batches = 4
batch_size, timesteps = 16, 8
batch_shape = (batch_size, timesteps)
embed_input_dim = 5
total_iterations = 0
self.model = self._make_model(batch_shape,
total_iterations,
embed_input_dim=embed_input_dim,
dense_constraint=1,
l2_reg=1e-4,
bidirectional=False,
sparse=True)
optimizer = self._make_optimizer(optimizer_name, self.model,
**optimizer_kw)
self.model.compile(optimizer,
loss='sparse_categorical_crossentropy')
X, Y = self._make_data(num_batches,
*batch_shape,
embed_input_dim=embed_input_dim,
sparse=True)
for batch_num in range(num_batches):
self.model.train_on_batch(X[batch_num], Y[batch_num])
self._test_save_load(self.model, X, optimizer_name, optimizer)
# util test
dc = {'lstm': 0, 'dense': 0}
fill_dict_in_order(dc, [1e-4, 2e-4])
AdamW(model=self.model, zero_penalties=True)
AdamW(model=self.model, weight_decays={'a': 0})
# cleanup
del self.model, optimizer
reset_seeds(reset_graph_with_backend=K)
cprint("\n<< {} MISC TEST PASSED >>\n".format(optimizer_name),
'green')
cprint("\n<< ALL MISC TESTS PASSED >>\n", 'green')
def aunet(pretrained_weights = None, input_size = (256,256,1)):
inputs = Input(input_size)
added1 = resblock(inputs, 64)
pool1 = MaxPooling2D(pool_size=(2, 2))(added1)
added2 = resblock(pool1, 128)
pool2 = MaxPooling2D(pool_size=(2, 2))(added2)
added3 = resblock(pool2, 256)
pool3 = MaxPooling2D(pool_size=(2, 2))(added3)
added4 = resblock(pool3, 512)
pool4 = MaxPooling2D(pool_size=(2, 2))(added4)
added5 = resblock(pool4, 1024)
merge6 = concat(added4, added5, 512)
added6 = Basic(merge6, 512)
merge7 = concat(added3, added6, 256)
added7 = Basic(merge7, 256)
merge8 = concat(added2, added7, 128)
added8 = Basic(merge8, 128)
merge9 = concat(added1, added8, 64)
added9 = Basic(merge9, 64)
conv10 = Conv2D(1, 1, activation = 'sigmoid')(added9)
model = Model(input = inputs, output = conv10)
#dl = balanced_cross_entropy(0.8)
dl=combined_loss()
opt = AdamW(lr=1e-3, model = model, use_cosine_annealing=True, total_iterations = 24)
#opt = keras.optimizers.Adam(lr=0.0001, amsgrad=True)
model.compile(optimizer = opt, loss = dl, metrics = ['accuracy', dice_score])
if(pretrained_weights):
model.load_weights(pretrained_weights)
return model
def test_misc(): # tests of non-main features to improve coverage
for optimizer_name in ['AdamW', 'NadamW', 'SGDW']:
cprint("<< TESTING {} OPTIMIZER >>".format(optimizer_name), 'blue')
reset_seeds()
optimizer_kw = {
'total_iterations': 0,
'decay': 1e-3,
'amsgrad': optimizer_name == 'AdamW',
'nesterov': optimizer_name == 'SGDW'
}
num_batches = 4
batch_size, timesteps = 16, 8
batch_shape = (batch_size, timesteps)
embed_input_dim = 5
# arbitrarily select SGDW for coverage testing
l1_reg = 1e-4 if optimizer_name == 'SGDW' else 0
l2_reg = 1e-4 if optimizer_name != 'SGDW' else 0
if optimizer_name == 'SGDW':
optimizer_kw.update(
dict(zero_penalties=False,
weight_decays={},
total_iterations=2,
momentum=0))
model = _make_model(batch_shape,
embed_input_dim=embed_input_dim,
dense_constraint=1,
l1_reg=l1_reg,
l2_reg=l2_reg,
bidirectional=False,
sparse=True)
optimizer = _make_optimizer(optimizer_name, model, **optimizer_kw)
model.compile(optimizer, loss='sparse_categorical_crossentropy')
X, Y = _make_data(num_batches,
*batch_shape,
embed_input_dim=embed_input_dim,
sparse=True)
for batch_num in range(num_batches):
model.train_on_batch(X[batch_num], Y[batch_num])
_test_save_load(model, X, optimizer_name, optimizer)
# util test
dc = {'lstm': 0, 'dense': 0}
fill_dict_in_order(dc, [1e-4, 2e-4])
AdamW(model=model, zero_penalties=False, total_iterations=2)
AdamW(model=model, weight_decays={'a': 0})
opt = AdamW(weight_decays={model.layers[1].weights[0].name: (0, 0)},
total_iterations=2)
model.compile(opt, loss='sparse_categorical_crossentropy')
model.train_on_batch(X[0], Y[0])
# cleanup
del model, optimizer
reset_seeds(reset_graph_with_backend=K)
try:
K_eval('x', K) # for coverage
except:
pass
cprint("\n<< {} MISC TEST PASSED >>\n".format(optimizer_name), 'green')
cprint("\n<< ALL MISC TESTS PASSED >>\n", 'green')
def deepSupervision(pretrained_weights=None, input_size=(256, 256, 3)):
inputs = Input(input_size)
#Encoder 1
added1 = encoder(inputs, 64)
pool1 = MaxPooling2D(pool_size=(2, 2))(added1)
#Encoder 2
added2 = encoder(pool1, 128)
pool2 = MaxPooling2D(pool_size=(2, 2))(added2)
att1 = attentionblock(added2, (2, 2))
#Encoder 3
added3 = encoder(pool2, 256)
pool3 = MaxPooling2D(pool_size=(2, 2))(added3)
att2 = attentionblock(added3, (4, 4))
#Encoder 4
added4 = encoder(pool3, 512)
pool4 = MaxPooling2D(pool_size=(2, 2))(added4)
att3 = attentionblock(added4, (8, 8))
#Encoder 5
added5 = encoder(pool4, 1024)
#Decoder 1
merge6 = decoder(added4, added5, 512)
added6 = Dropout(0.5)(merge6)
#added6 = merge6
att4 = attentionblock(added6, (8, 8))
#Decoder 2
merge7 = decoder(added3, added6, 256)
added7 = Dropout(0.5)(merge7)
#added7 = merge7
att5 = attentionblock(added7, (4, 4))
#Decoder 3
merge8 = decoder(added2, added7, 128)
added8 = Dropout(0.5)(merge8)
#added8 = merge8
att6 = attentionblock(added8, (2, 2))
#Decoder 4
added9 = decoder(added1, added8, 64)
added9 = Dropout(0.5)(added9)
added9 = concatenate([added9, att1, att2, att3, att4, att5, att6], axis=3)
added10 = Conv2D(1, 1, activation='sigmoid', padding='same')(added9)
model = Model(input=inputs, output=added10)
dl = composite_loss(att1, att2, att3, att4, att5, att6, added9)
opt = AdamW(lr=1e-3,
model=model,
use_cosine_annealing=True,
total_iterations=24)
model.compile(optimizer=opt, loss=dl, metrics=['accuracy', dice_score])
if (pretrained_weights):
model.load_weights(pretrained_weights)
return model
#%%############################################################################
ipt = Input(shape=(120, 4))
x = LSTM(60,
activation='relu',
name='lstm_1',
kernel_regularizer=l1(1e-4),
recurrent_regularizer=l2(2e-4))(ipt)
out = Dense(1, activation='sigmoid', kernel_regularizer=l1_l2(1e-4, 2e-4))(x)
model = Model(ipt, out)
lr_multipliers = {'lstm_1': 0.5}
optimizer = AdamW(lr=1e-4,
model=model,
lr_multipliers=lr_multipliers,
use_cosine_annealing=True,
total_iterations=24)
model.compile(optimizer, loss='binary_crossentropy')
#%%############################################################################
eta_history = []
lr_history = []
for epoch in range(3):
for iteration in range(24):
x = np.random.rand(10, 120, 4) # dummy data
y = np.random.randint(0, 2, (10, 1)) # dummy labels
loss = model.train_on_batch(x, y)
eta_history.append(K_eval(model.optimizer.eta_t, K))
lr_history.append(K_eval(model.optimizer.lr_t, K))
print("Iter {} loss: {}".format(iteration + 1, "%.3f" % loss))