def build_model(hp):
params = hp.copy()
params['e_dim'] = params['dim']
params['r_dim'] = params['dim']
params['name'] = 'embedding_model'
embedding_model = models[params['embedding_model']]
embedding_model = embedding_model(**params)
triple = Input((3, ))
ftriple = Input((3, ))
inputs = [triple, ftriple]
score = embedding_model(triple)
fscore = embedding_model(ftriple)
loss_function = loss_function_lookup(params['loss_function'])
loss = loss_function(score, fscore, params['margin'] or 1, 1)
model = Model(inputs=inputs, outputs=loss)
model.add_loss(loss)
model.compile(optimizer=Adam(learning_rate=ExponentialDecay(
params['learning_rate'], decay_steps=100000, decay_rate=0.96)),
loss=None)
return model
def get_optimizer(
optimizer_key: str,
learning_rate: float,
learning_rate_decay: float,
learning_rate_decay_steps: int,
) -> tf_optimizers.Optimizer:
# Define an exponential learning rate decay schedule
learning_rate_schedule = ExponentialDecay(
learning_rate,
decay_steps=learning_rate_decay_steps,
decay_rate=learning_rate_decay,
staircase=True,
)
if optimizer_key == OptimizerKey.ADAM:
return tf_optimizers.Adam(learning_rate=learning_rate_schedule)
elif optimizer_key == OptimizerKey.NADAM:
return tf_optimizers.Nadam(learning_rate=learning_rate_schedule)
elif optimizer_key == OptimizerKey.ADAGRAD:
return tf_optimizers.Adagrad(learning_rate=learning_rate_schedule)
elif optimizer_key == OptimizerKey.SGD:
return tf_optimizers.SGD(learning_rate=learning_rate_schedule)
elif optimizer_key == OptimizerKey.RMS_PROP:
return tf_optimizers.RMSprop(learning_rate=learning_rate_schedule)
else:
raise ValueError("illegal Optimizer key: " + optimizer_key)
def siamese_model_ResNet(model_type):
if model_type == 'ResNet_50':
LR = 0.001
base_model = ResNet_50()
if model_type == 'ResNet_18':
LR = 0.0001
base_model = ResNet_18()
imgA = Input(shape=(64, 64, 3))
imgB = Input(shape=(64, 64, 3))
featsA = base_model(imgA)
featsB = base_model(imgB)
distance = Lambda(euclidean_distance, name='compute_ED')([featsA, featsB])
model = Model(inputs=[imgA, imgB], outputs=distance, name='Contrastive')
# model.load_weights('saved_models/checkpoints/' + model_name)
initial_learning_rate = LR
lr_schedule = ExponentialDecay(initial_learning_rate,
decay_steps=1562,
decay_rate=0.90,
staircase=True)
model.compile(loss=contrastive_loss,
optimizer=keras.optimizers.Adam(lr_schedule),
metrics=[accuracy1]) #, recall_m, precision_m])
model.summary()
return model
def __init__(self, num_scales, num_iters, max_size, min_size, scale_factor,
learning_rate, checkpoint_dir, debug):
self.num_scales = num_scales
self.num_iters = num_iters
self.num_filters = [
32 * pow(2, (scale // 4)) for scale in range(self.num_scales)
] # num_filters double for every 4 scales
self.max_size = max_size
self.min_size = min_size
self.scale_factor = scale_factor
self.noise_amp_init = 0.1
self.checkpoint_dir = checkpoint_dir
self.G_dir = self.checkpoint_dir + '/G'
self.D_dir = self.checkpoint_dir + '/D'
self.learning_schedule = ExponentialDecay(
learning_rate, decay_steps=4800, decay_rate=0.1,
staircase=True) # 1600 * 3 steps
self.build_model()
self.debug = debug
if self.debug:
self.create_summary_writer()
self.create_metrics()
def _compile_model(
self,
loss_function: Callable,
ground_distance_path: Path,
**loss_function_kwargs
):
if loss_function == self_guided_earth_mover_distance:
self.emd_weight_head_start = EmdWeightHeadStart()
self.ground_distance_manager = GroundDistanceManager(ground_distance_path)
lr_schedule = ExponentialDecay(
self.learning_rate,
decay_steps=429,
decay_rate=0.995
)
self.compile(
loss=loss_function(
model=self,
**loss_function_kwargs
),
optimizer=self._OPTIMIZER(
learning_rate=lr_schedule,
nesterov=True,
momentum=self._OPTIMIZER_MOMENTUM
),
metrics=self._METRICS,
run_eagerly=True
)
def get_lr_scheduler(learning_rate, decay_type, decay_steps):
if decay_type:
decay_type = decay_type.lower()
if decay_type == None:
lr_scheduler = learning_rate
elif decay_type == 'cosine':
lr_scheduler = CosineDecay(
initial_learning_rate=learning_rate,
decay_steps=decay_steps,
alpha=0.2) # use 0.2*learning_rate as final minimum learning rate
elif decay_type == 'exponential':
lr_scheduler = ExponentialDecay(initial_learning_rate=learning_rate,
decay_steps=decay_steps,
decay_rate=0.9)
elif decay_type == 'polynomial':
lr_scheduler = PolynomialDecay(initial_learning_rate=learning_rate,
decay_steps=decay_steps,
end_learning_rate=learning_rate / 100)
elif decay_type == 'piecewise_constant':
#apply a piecewise constant lr scheduler, including warmup stage
boundaries = [500, int(decay_steps * 0.9), decay_steps]
values = [
0.001, learning_rate, learning_rate / 10., learning_rate / 100.
]
lr_scheduler = PiecewiseConstantDecay(boundaries=boundaries,
values=values)
else:
raise ValueError('Unsupported lr decay type')
return lr_scheduler
def define_model():
model = Sequential()
model.add(
Conv2D(32, (3, 3),
activation='relu',
kernel_initializer='he_uniform',
padding='same',
input_shape=(32, 32, 3)))
model.add(BatchNormalization())
model.add(
Conv2D(32, (3, 3),
activation='relu',
kernel_initializer='he_uniform',
padding='same'))
model.add(BatchNormalization())
model.add(MaxPooling2D((2, 2)))
model.add(Dropout(0.2))
model.add(
Conv2D(64, (3, 3),
activation='relu',
kernel_initializer='he_uniform',
padding='same'))
model.add(BatchNormalization())
model.add(
Conv2D(64, (3, 3),
activation='relu',
kernel_initializer='he_uniform',
padding='same'))
model.add(BatchNormalization())
model.add(MaxPooling2D((2, 2)))
model.add(Dropout(0.3))
model.add(
Conv2D(128, (3, 3),
activation='relu',
kernel_initializer='he_uniform',
padding='same'))
model.add(BatchNormalization())
model.add(
Conv2D(128, (3, 3),
activation='relu',
kernel_initializer='he_uniform',
padding='same'))
model.add(BatchNormalization())
model.add(MaxPooling2D((2, 2)))
model.add(Dropout(0.4))
model.add(Flatten())
model.add(Dense(128, activation='relu', kernel_initializer='he_uniform'))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))
# compile model
lr_schedule = ExponentialDecay(initial_learning_rate=0.01,
decay_steps=10000,
decay_rate=0.9)
opt = SGD(learning_rate=lr_schedule)
model.compile(optimizer=opt,
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
def getScheduler(sched, lr=1e-3):
if sched == 'poly':
return PolynomialDecay(lr, 3000, 0.99)
elif sched == 'exp':
return ExponentialDecay(lr, 1000)
else:
return lr
def train_from_cache(
*,
architecture: str = "BiT",
size: int = 256,
augmentation: int = 1,
epochs: int = 20,
batch_size: int = 32,
learning_rate: float = 0.001,
lr_decay_rate: float = 0.99,
lr_decay_steps: int = 5e2,
**model_params,
) -> float:
"""
Trains the model using embeddings that were previously cached.
"""
train_data, dev_data = load_embeddings(architecture, size, augmentation)
# Make and init the wandb run.
wandb.init(project="Plant Pathology", reinit=True)
wandb.config.update(
{
"architecture": architecture,
"epochs": epochs,
"batch_size": batch_size,
"learning_rate": learning_rate,
"lr_decay_rate": lr_decay_rate,
"lr_decay_steps": lr_decay_steps,
"augmentation": augmentation,
"size": size,
}
)
model = PathologistModel(nclasses=constants.NCLASSES, **model_params)
model.compile(
optimizer=Adam(
learning_rate=ExponentialDecay(learning_rate, lr_decay_steps, lr_decay_rate)
),
loss="categorical_crossentropy",
metrics=["categorical_accuracy"],
)
model.fit(
train_data["X"],
train_data["y"],
batch_size=batch_size,
epochs=epochs,
validation_data=(dev_data["X"], dev_data["y"]),
callbacks=[WandbCallback(save_model=False)],
)
# Log the scores
train_loss, train_acc = model.evaluate(train_data["X"], train_data["y"])
_, dev_acc = model.evaluate(dev_data["X"], dev_data["y"])
wandb.run.summary.update(
{"final_train_loss": train_loss, "final_train_acc": train_acc}
)
wandb.join()
return dev_acc
def test_constant_lr(self):
"""Test a constant learning rate"""
lrs = ExponentialDecay(
initial_learning_rate=0.1,
decay_steps=10000000,
decay_rate=1.0,
)
for i in range(25):
assert round(float(lrs(i).numpy()), 5) == 0.1
def __init__(self, model, ckpt_path, args):
super().__init__(model=model,
loss=MeanAbsoluteError(),
learning_rate=ExponentialDecay(args.lr_init,
args.lr_decay_step,
args.lr_decay_ratio,
staircase=True),
checkpoint_path=ckpt_path,
args=args)
def get_optimizer(
optimizer_key: str,
learning_rate: float,
learning_rate_decay: float = 1.0,
learning_rate_decay_steps: int = 1000000,
gradient_clip_value: float = 1000000,
) -> tf_optimizers.Optimizer:
"""
This function defines the optimizer used by ml4ir.
Users have the option to define an ExponentialDecay learning rate schedule
Arguments:
optimizer_key: string optimizer name to be used as defined under ml4ir.base.config.keys.OptimizerKey
learning_rate: floating point learning rate for the optimizer
learning_rate_decay: floating point rate at which the learning rate will be decayed every learning_rate_decay_steps
learning_rate_decay_steps: int representing number of iterations after which learning rate will be decreased exponentially
gradient_clip_value: float value representing the clipvalue for gradient updates. Not setting this to a reasonable value based on the model will lead to gradient explosion and NaN losses.
References:
https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/Optimizer
https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/schedules/ExponentialDecay
FIXME:
Define all arguments overriding tensorflow defaults in a separate file
for visibility with ml4ir users
"""
# Define an exponential learning rate decay schedule
learning_rate_schedule = ExponentialDecay(
learning_rate,
decay_steps=learning_rate_decay_steps,
decay_rate=learning_rate_decay,
staircase=True,
)
if optimizer_key == OptimizerKey.ADAM:
return tf_optimizers.Adam(
learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value
)
elif optimizer_key == OptimizerKey.NADAM:
return tf_optimizers.Nadam(
learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value
)
elif optimizer_key == OptimizerKey.ADAGRAD:
return tf_optimizers.Adagrad(
learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value
)
elif optimizer_key == OptimizerKey.SGD:
return tf_optimizers.SGD(
learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value
)
elif optimizer_key == OptimizerKey.RMS_PROP:
return tf_optimizers.RMSprop(
learning_rate=learning_rate_schedule, clipvalue=gradient_clip_value
)
else:
raise ValueError("illegal Optimizer key: " + optimizer_key)
def __init__(self, env: '', episodes=1000, alpha=0.01, gamma=0.9, alpha_decay_rate=0.9):
self.env = Environment(env=env)
self.episodes = episodes
self.lr = ExponentialDecay(alpha, episodes, alpha_decay_rate)
self.optimizer = Adam(self.lr)
self.action_count, self.states_count = self.env.spaces_count()
self.gamma = gamma
self._net = ReinforcePolicyNet(action_count=self.action_count, states_count=self.states_count)
self._model = ReinforcePolicyModel(self._net)
self._agent = ReinforcePolicyAgent(env=self.env, model=self._model, gamma=gamma)
self.huber_loss = Huber(reduction=tf.keras.losses.Reduction.SUM)
def nvidia():
"""
Implementation of Nvidia's End-to-End Learning model for Self-driving cars
"""
global X_train, y_train
# Model Design
inputs = Input(shape=(160,320,3))
cropped = Cropping2D(cropping=((64, 0), (0, 0)))(inputs)
resized_input = Lambda(lambda image: tf.image.resize(image, (66,200)))(cropped)
normalize_layer = LayerNormalization(axis=1)(resized_input)
conv1 = Conv2D(filters=24, kernel_size=5, strides=(2,2), activation='relu')(normalize_layer)
conv2 = Conv2D(filters=36, kernel_size=5, strides=(2,2), activation='relu')(conv1)
conv3 = Conv2D(filters=48, kernel_size=5, strides=(2,2), activation='relu')(conv2)
conv4 = Conv2D(filters=64, kernel_size=3, activation='relu')(conv3)
conv5 = Conv2D(filters=64, kernel_size=3, activation='relu')(conv4)
flatten = Flatten()(conv5)
dense1 = Dense(100,activation='relu')(flatten)
dense2 = Dense(50,activation='relu')(dense1)
dense3 = Dense(10,activation='relu')(dense2)
out = Dense(1, activation='linear')(dense3)
# Specifications and training
checkpoint = ModelCheckpoint(filepath="./ckpts/model_nvidia.h5", monitor='val_loss', save_best_only=True)
stopper = EarlyStopping(monitor='val_loss', min_delta=0.0003, patience = 10)
lr_schedule = ExponentialDecay(initial_learning_rate=0.0001, decay_steps=100000, decay_rate=0.95)
optimizer = Adam(learning_rate=lr_schedule)
loss = Huber(delta=0.5, reduction="auto", name="huber_loss")
t2 = time()
model = Model(inputs=inputs, outputs=out)
model.compile(loss = loss, optimizer = optimizer)
result = model.fit(X_train, y_train, validation_split = 0.2, shuffle = True,
epochs = 100, callbacks=[checkpoint, stopper])
# Visualization of loss variations across epochs
plt.plot(result.history['loss'])
plt.plot(result.history['val_loss'])
plt.title('Huber Loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Training set', 'Validation set'], loc = 'upper right')
plt.savefig('loss.png')
plt.show()
print("Time taken to train: {:.2f}s".format(time()-t2))
model.load_weights('./ckpts/model_nvidia.h5')
model.save('model.h5')
def __init__(self, config: MuZeroConfig, storage: SharedStorage,
replay_buffer: ReplayBuffer):
self.config = config
self.storage = storage
self.replay_buffer = replay_buffer
self.summary = create_summary(name="leaner")
self.metrics_loss = Mean(f'leaner-loss', dtype=tf.float32)
self.network = Network(self.config)
self.lr_schedule = ExponentialDecay(
initial_learning_rate=self.config.lr_init,
decay_steps=self.config.lr_decay_steps,
decay_rate=self.config.lr_decay_rate)
self.optimizer = Adam(learning_rate=self.lr_schedule)
def exponential(learn_rate):
if not isinstance(learn_rate, (list, tuple)):
raise ValueError(
"Exponential decay requres three values: starting learning rate, steps, and decay factor"
)
start_lr = learn_rate[0]
steps = learn_rate[1]
decay = learn_rate[2]
learning_rate_sced = ExponentialDecay(start_lr,
steps,
decay,
staircase=True)
return learning_rate_sced
def transfer_train(
*,
train_set: str,
size: int = 256,
epochs: int = 20,
batch_size: int = 32,
learning_rate: float = 0.001,
lr_decay_rate: float = 0.99,
lr_decay_steps: int = 5e2,
make_submission: bool = False,
**model_params,
):
"""
Trains a new head on top of a transfer model. No fine tuning of the transfer
model is conducted. Transfer model embeddings are computed once at the
beginning of the training run. If `make_submission==True`, returns this model's
scores on the test set, along with the corresponding image ids.
"""
# Make and init the wandb run.
wandb.init(project="Plant Pathology", reinit=True)
wandb.config.update({
"epochs": epochs,
"batch_size": batch_size,
"learning_rate": learning_rate,
"lr_decay_rate": lr_decay_rate,
"lr_decay_steps": lr_decay_steps,
"size": size,
})
model = TransferModel(constants.NCLASSES, size, batch_size, **model_params)
model.compile(
optimizer=Adam(learning_rate=ExponentialDecay(
learning_rate, lr_decay_steps, lr_decay_rate)),
loss="categorical_crossentropy",
metrics=["categorical_accuracy"],
)
# Train the model (just the new layers on top of the transfer model)
model.fit_head(
train_set,
"dev",
epochs=epochs,
callbacks=[WandbCallback(save_model=False)],
)
# Log the scores
wandb.join()
if make_submission:
return model.predict_on_test()
def get_network(learning_rate=0.001,
momentum=0.5,
decay_rate=0.98,
use_sgd=True,
kernel_regularizer_conv=0.01,
kernel_regularizer_dense=0.0001,
**kwargs):
convolutional_network = _get_convolutional_network(
kernel_regularizer_conv, kernel_regularizer_dense)
input_image_1 = Input(input_shape)
input_image_2 = Input(input_shape)
output_image_1 = convolutional_network(input_image_1)
output_image_2 = convolutional_network(input_image_2)
l1_distance_layer = Lambda(lambda tensors: K.abs(tensors[0] - tensors[1]))
l1_distance = l1_distance_layer([output_image_1, output_image_2])
prediction = Dense(units=1,
activation='sigmoid',
kernel_initializer=RandomNormal(mean=0, stddev=0.01),
bias_initializer=RandomNormal(mean=0.5,
stddev=0.01))(l1_distance)
model = Model(inputs=[input_image_1, input_image_2], outputs=prediction)
if use_sgd:
optimizer = SGD(learning_rate=ExponentialDecay(learning_rate, 100000,
decay_rate),
momentum=momentum)
else:
optimizer = Adam(
learning_rate=ExponentialDecay(learning_rate, 100000, decay_rate))
model.compile(loss='binary_crossentropy',
metrics=['binary_accuracy'],
optimizer=optimizer)
return model
def train(args):
awv = audio_array(
args.content) #get waveform array from folder containing wav files
aspec = tospec(awv) #get spectrogram array
adata = splitcut(aspec) #split spectrogams to fixed length
bwv = audio_array(args.style)
bspec = tospec(bwv)
bdata = splitcut(bspec)
dsa = tf.data.Dataset.from_tensor_slices(adata).repeat(50).map(
lambda x: proc(x, hp),
num_parallel_calls=tf.data.experimental.AUTOTUNE).shuffle(10000).batch(
hp.bs, drop_remainder=True)
dsb = tf.data.Dataset.from_tensor_slices(bdata).repeat(50).map(
lambda x: proc(x, hp),
num_parallel_calls=tf.data.experimental.AUTOTUNE).shuffle(10000).batch(
hp.bs, drop_remainder=True)
dataset = tf.data.Dataset.zip((dsa, dsb))
model = Adverserial()
gen_opt = Adam(
ExponentialDecay(hp.lr, decay_steps=hp.epochs * 2, decay_rate=0.1),
hp.opt_alpha)
disc_opt = Adam(
ExponentialDecay(hp.lr, decay_steps=hp.epochs * 2, decay_rate=0.1),
hp.opt_alpha)
model.compile(gen_opt, disc_opt, adv_loss)
# Log metrics with wandb
callbacks = [
TrainCallback(model, aspec),
EarlyStopping(monitor="loss_g", patience=50)
]
model.fit(dataset, epochs=hp.epochs, callbacks=callbacks)
def main(args):
timestamp = int(time.time())
logdir = os.path.join(args.volumedir,
datetime.datetime.today().strftime('%Y%m%d'),
args.logdir)
if not os.path.isdir(logdir):
os.makedirs(logdir)
snapdir = os.path.join(args.volumedir,
datetime.datetime.today().strftime('%Y%m%d'),
"progess_snaps")
if not os.path.isdir(snapdir):
os.makedirs(snapdir)
hdlr = logging.FileHandler(
os.path.join(logdir, "training_output_%d.log" % timestamp))
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
logger.setLevel(logging.INFO)
checkpointdir = os.path.join(args.volumedir,
datetime.datetime.today().strftime('%Y%m%d'),
args.checkpointdir)
if not os.path.isdir(checkpointdir):
os.makedirs(checkpointdir)
# Dynamically load modelBuilder class
moduleName, klassName = args.modelbuilder.split(".")
mod = __import__('models.%s' % moduleName, fromlist=[klassName])
klass = getattr(mod, klassName)
modelBuilder = klass(args)
optimizer_map = {"adam": Adam, "rmsprop": RMSprop, "sgd": SGD}
optimizer = optimizer_map[
args.optimizer] if args.optimizer in optimizer_map.keys() else RMSprop
lr_decay = ExponentialDecay(initial_learning_rate=args.learningrate,
decay_rate=args.decayrate,
decay_steps=args.decaysteps)
opt = optimizer(learning_rate=lr_decay, clipvalue=3)
model = modelBuilder.get_model()
C, S, G = modelBuilder.get_train_data()
for i in range(1, args.numepochs + 1):
loss, grads = modelBuilder.compute_loss_and_grads(model, C, S, G)
opt.apply_gradients([(grads, G)])
if i % 100 == 0:
logger.info("Iteration: %d, Loss: %.2f" % (i, loss))
modelBuilder.save_generated_img(snapdir, G.numpy(), i, timestamp)
def getModel(lag):
model = Sequential([
Dense(16, activation='relu'),
Dense(32, activation='relu'),
Dense(16, activation='relu'),
Dense(1, activation='linear')
])
model.build(input_shape=(None, lag))
model.compile(optimizer=Adam(ExponentialDecay(1e-3, 50, 0.9)),
loss=BinaryCrossentropy(from_logits=True))
return model
def test_exp_lr(self):
"""Test a exponential learning rate"""
gold = [0.10000000149011612,0.09486832469701767,0.08999999612569809,0.08538150042295456,0.08099999278783798,
0.07684334367513657,0.07289998978376389,0.06915900856256485,0.06560999155044556,0.062243103981018066,
0.05904899165034294,0.05601879581809044,0.05314409360289574,0.050416912883520126,0.04782968387007713,
0.045375220477581024,0.04304671287536621,0.04083769768476486,0.03874203935265541,0.036753926426172256,
0.03486783429980278,0.03307853266596794,0.03138105198740959,0.029770677909255028,0.028242945671081543]
lrs = ExponentialDecay(
initial_learning_rate=0.1,
decay_steps=2,
decay_rate=0.9,
)
scheduler = [lrs(i) for i in range(25)]
self.assertTrue(self.compare_lr_values(scheduler, gold))
def compile_model(input_timesteps: int = 1,
output_timesteps: int = 1,
categories: int = 3,
loss_function: str = "categorical_crossentropy",
initial_learning_rate: float = 0.001,
decay_steps: int = 1000,
decay_rate: float = 0.95,
metrics: list = [],
**kwargs):
model = Sequential()
model.add(Conv3D(filters=16, kernel_size=(3, 3, 3), padding="same"))
model.add(
Bidirectional(
ConvLSTM2D(filters=16,
kernel_size=(3, 3),
return_sequences=True,
padding="same",
dropout=0.1)))
model.add(BatchNormalization())
model.add(
Bidirectional(
ConvLSTM2D(filters=32,
kernel_size=(3, 3),
return_sequences=True,
padding="same",
dropout=0.5)))
model.add(BatchNormalization())
model.add(
Bidirectional(
ConvLSTM2D(filters=32,
kernel_size=(3, 3),
return_sequences=True,
padding="same",
dropout=0.5)))
model.add(BatchNormalization())
model.add(
Conv3D(filters=categories,
kernel_size=(input_timesteps - output_timesteps + 1, 1, 1),
activation="softmax",
padding="valid"))
optimizer = Adam(learning_rate=ExponentialDecay(
initial_learning_rate=initial_learning_rate,
decay_steps=decay_steps,
decay_rate=decay_rate))
model.compile(loss=loss_function, optimizer=optimizer, metrics=metrics)
return model
def get_lr_scheduler(learning_rate, decay_type, decay_steps):
if decay_type:
decay_type = decay_type.lower()
if decay_type == None:
lr_scheduler = learning_rate
elif decay_type == 'cosine':
lr_scheduler = CosineDecay(initial_learning_rate=learning_rate, decay_steps=decay_steps)
elif decay_type == 'exponential':
lr_scheduler = ExponentialDecay(initial_learning_rate=learning_rate, decay_steps=decay_steps, decay_rate=0.9)
elif decay_type == 'polynomial':
lr_scheduler = PolynomialDecay(initial_learning_rate=learning_rate, decay_steps=decay_steps, end_learning_rate=learning_rate/100)
else:
raise ValueError('Unsupported lr decay type')
return lr_scheduler
def getExtremeModel(optimizerExtremeParam, lag):
modelExtreme = Sequential([
Dense(16, activation='relu'),
Dense(32, activation='relu'),
Dense(64, activation='relu'),
Dense(32, activation='relu'),
Dense(16, activation='relu'),
Dense(1, activation='sigmoid'),
])
modelExtreme.build(input_shape=(None, lag))
modelExtreme.compile(optimizer=Adam(
ExponentialDecay(*optimizerExtremeParam)),
loss=tf.losses.MeanSquaredError())
return modelExtreme
def getNormalModel(optimizerNormalParam, lag):
modelNormal = Sequential([
Dense(16, activation='relu'),
Dense(32, activation='relu'),
Dense(64, activation='relu'),
Dense(32, activation='relu'),
Dense(16, activation='relu'),
Dense(1, activation='linear'),
])
modelNormal.build(input_shape=(None, lag))
modelNormal.compile(optimizer=Adam(
ExponentialDecay(*optimizerNormalParam)),
loss=MeanSquaredError())
return modelNormal
def train_model(model, x, y, task):
date = datetime.now().strftime("%d-%m-%Y-%H-%M-%S")
checkpoints_path = f'/opt/ml/model/checkpoints/{task}/{date}'
checkpoints_file_format = 'checkpoints.{epoch:02d}-{val_loss:.2f}.hdf5'
checkpoint_callback = ModelCheckpoint(
posixpath.join(checkpoints_path, checkpoints_file_format))
L = len(x)
split_ration = 0.8
train_indexes = random.sample(range(L), int(L * split_ration))
validation_indexes = list(set(range(L)) - set(train_indexes))
x_train = x[train_indexes]
y_train = y[train_indexes]
x_val = x[validation_indexes]
y_val = y[validation_indexes]
generator = ImageDataGenerator(rotation_range=30,
zoom_range=0.20,
fill_mode="nearest",
shear_range=0.20,
horizontal_flip=False,
width_shift_range=0.2,
height_shift_range=0.2)
lr_schedule = ExponentialDecay(initial_learning_rate=0.01,
decay_steps=100,
decay_rate=0.9,
staircase=False)
model.compile(loss='categorical_crossentropy',
optimizer=Adam(learning_rate=lr_schedule),
metrics=['accuracy'])
model.fit(generator.flow(x_train, y_train, batch_size=32),
steps_per_epoch=len(x_train) // 32,
verbose=2,
epochs=500,
shuffle=True,
validation_data=(x_val, y_val),
callbacks=[
WandbCallback(), checkpoint_callback,
LearningRateCallback(model)
])
model.save('/opt/ml/model')
def get_model(n_categories,
net_spec,
learning_rate,
steps_per_epoch,
lr_decay_period=5,
staircase=True):
"""
Generates a sequential model with the provided parameters
Args:
n_categories: number of possible outcomes for the model to predict
net_spec: a tuple with the numbers of nodes in each layer. For single layer must be like (N,)
learning_rate: initial learning rate for the optimization process
steps_per_epoch: total steps taken in a single epoch
lr_decay_period: number of epochs before the learning rate becomes halved
staircase: boolean. Whether to use a staircase pattern for the learning rate decay
Returns:
A corresponding keras.Sequetial model
"""
model = keras.Sequential([
layers.Dense(net_spec[0],
input_dim=13,
activation='relu',
name='hidden_layer_0'),
] + [
layers.Dense(layer_size, activation='relu', name=f'hidden_layer_{i+1}')
for i, layer_size in enumerate(net_spec[1:])
] + [
layers.Dense(n_categories, activation='softmax', name='output_layer')
])
decay_steps = steps_per_epoch * lr_decay_period
lr_schedule = ExponentialDecay(learning_rate,
decay_steps=decay_steps,
decay_rate=0.5,
staircase=staircase)
opt = keras.optimizers.Adam(learning_rate=lr_schedule)
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['categorical_accuracy'])
return model
def create_model(input_shape=10,
output_shape=10,
layers=[50],
activation='relu',
optimizer='adam',
learning_rate=0.001,
dropout=True,
decay=0.9,
decay_steps=10000,
batch_norm=True):
model = Sequential(name=str(time.time()))
model.add(Input(input_shape))
for l in layers:
model.add(Dense(l))
if batch_norm:
model.add(BatchNormalization())
model.add(Activation(activation=activation))
if dropout:
model.add(Dropout(0.5))
model.add(Dense(output_shape))
# Compile model
lr_schedule = ExponentialDecay(initial_learning_rate=learning_rate,
decay_steps=decay_steps,
decay_rate=decay)
opt = optimizers.get({
'class_name': optimizer,
'config': {
'learning_rate': lr_schedule
}
})
model.compile(loss='mse', optimizer=opt, metrics=[r2_metric])
return model
def training(self):
def _loss(y_true, y_pred):
print('y_pred.shape:', y_pred.shape)
print('y_true.shape: ', y_true.shape)
labels = tf.cast(tf.squeeze(y_true), tf.int64)
return tf.reduce_mean(
input_tensor=tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=y_pred, labels=labels))
print('Executing eagerly? ', tf.executing_eagerly())
self.build_graph()
learning_rate = ExponentialDecay(initial_learning_rate=LEARNING_RATE,
decay_steps=2000,
decay_rate=0.97,
staircase=True)
optimizer = optimizers.Adam(learning_rate=learning_rate)
#self.model.compile(loss="categorical_crossentropy", optimizer=optimizer, metrics=["accuracy"])
self.model.compile(loss=_loss,
optimizer=optimizer,
metrics=["accuracy"])
self.model.run_eagerly = True
self.model.summary()
#keras.utils.plot_model(self.model, show_shapes=True)
checkpoint_callback = callbacks.ModelCheckpoint(MODEL_NAME + ".h5",
save_freq='epoch')
time_callback = TimeHistory()
run_logdir = self.get_run_logdir()
tensorboard_callback = callbacks.TensorBoard(run_logdir)
data = Data()
train_dataset = data.prepare_data_for_training(Data.DATA_TRAINING,
batch_size)
history = self.model.fit(train_dataset,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[time_callback])
#score = self.model.evaluate(X_test, y_test)
self.save_model()