def predict(title, description):
# Load model and weights
model = MyModel()
model.load_weights(model_name)
# Clean the description
des = clean_data_for_overview(description)
# Predict
genre = model.predict(np.array([des]))
# Prepare the output
res = r'''{
"title": "%s",
"description": "%s",
"genre": "%s"
}''' % (title, description, genre[0])
print(res)
def test_model(ds_name, encoder, paths, categorical=False):
"""The main function for executing network testing. It loads the specified
dataset iterator and optimized saliency model. By default, when no model
checkpoint is found locally, the pretrained weights will be downloaded.
Args:
ds_name (str): Denotes the dataset that was used during training.
encoder (str): the name of the encoder want to be used to predict.
paths (dict, str): A dictionary with all path elements.
"""
w_filename_template = "/%s_%s_%s_weights.h5" # [encoder]_[ds_name]_weights.h5
(test_ds, n_test) = data.load_test_dataset(ds_name, paths["data"], categorical)
print(">> Preparing model with encoder %s..." % encoder)
model = MyModel(encoder, ds_name, "test")
weights_path = paths["weights"] + w_filename_template % (encoder, ds_name, loss_fn_name)
if os.path.exists(weights_path):
print("Weights are loaded!\n %s"%weights_path)
else:
download.download_pretrained_weights(paths["weights"], encoder, ds_name, loss_fn_name)
model.load_weights(weights_path)
del weights_path
print(">> Start predicting using model trained on %s..." % ds_name.upper())
results_path = paths["results"] + "%s/%s/%s/" % (ds_name, encoder, loss_fn_name)
# Preparing progbar
test_progbar = Progbar(n_test)
for test_images, test_ori_sizes, test_filenames in test_ds:
pred = test_step(test_images, model)
for pred, filename, ori_size in zip(pred, test_filenames.numpy(), test_ori_sizes):
img = data.postprocess_saliency_map(pred, ori_size, as_image=True)
tf.io.write_file(results_path + filename.decode("utf-8"), img)
test_progbar.add(test_images.shape[0])
# lr ExponentialDecay
lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(
config['learning_rate'],
decay_steps=config['decay_steps'],
decay_rate=config['decay_rate'])
model.compile(
optimizer=tf.keras.optimizers.Adam(learning_rate=lr_schedule),
loss=Loss(),
metrics=[PSNR(), SSIM()],
)
# resume checkpoint
if config['resume']:
model.build((None, None, None, 3))
model.load_weights(checkpoint_path)
# save the best model checkpoint
model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
filepath=checkpoint_path,
save_weights_only=True,
monitor='val_loss',
mode='min',
save_best_only=True)
# tensorboard visulization
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir,
histogram_freq=1)
# model fit
model.fit(trainset,
# COLORMAP = [[0, 0, 255], [0, 255, 0]]
# cm = np.array(COLORMAP).astype(np.uint8)
# pred = np.argmax(np.array(pred), axis=2)
# pred_val = cm[pred]
# cv2.imwrite(os.path.join("data",filename.split("/")[-1]), pred_val)
# print(os.path.join("data",filename.split("/")[-1])+"finished")
test_dataset = tf.data.Dataset.from_generator(
test_generator, tf.float32, tf.TensorShape([None, None, None]))
test_dataset = test_dataset.batch(5)
#model = DeepLabV3Plus(image_shape[0], image_shape[1], nclasses=4)
model = MyModel(4)
model.load_weights(weight_path + 'fcn_20191021.ckpt')
test_list_dir = os.listdir(test_dir)
test_list_dir.sort()
test_filenames = [test_dir + filename for filename in test_list_dir]
for filename in test_filenames:
image = scipy.misc.imresize(scipy.misc.imread(filename),
image_shape) # image dim=[h, w, channel]
#image = image/255
#image_yuv = cv2.cvtColor(image, cv2.COLOR_RGB2YUV)
#image_yuv[:, :, 0] = cv2.equalizeHist(image_yuv[:, :, 0])
#image = cv2.cvtColor(image_yuv, cv2.COLOR_YUV2RGB)
image = image[np.newaxis, :, :, :].astype("float32")
class ModelHelper:
def __init__(self):
self.model = MyModel()
# change the learning rate rate to start training from scratch
self.optimizer = tf.keras.optimizers.RMSprop(learning_rate=0.00001)
@tf.function
def train_step(self, images, labels, seq_len):
with tf.GradientTape() as tape:
predictions = self.model(images)
loss = tf.math.reduce_mean(
tf.nn.ctc_loss(labels=labels,
logits=predictions,
logit_length=seq_len,
label_length=None,
blank_index=-1))
gradients = tape.gradient(loss, self.model.trainable_variables)
self.optimizer.apply_gradients(
zip(gradients, self.model.trainable_variables))
return loss
def train(self, loader):
# load previously trained weights
self.load_weights(FilePaths.fnWeight)
# define the number of epochs to run
epochs = 1
for epoch in range(epochs):
loader.train_set()
while loader.has_next():
iter_info = loader.get_iterator_info()
batch = loader.get_next()
labels = to_sparse(batch.gtTexts)
labels = tf.SparseTensor(labels[0], labels[1], labels[2])
sequence_lengths = tf.cast(tf.fill([MyModel.batch_size],
MyModel.max_text_len),
dtype=tf.int32)
loss = self.train_step(tf.cast(batch.imgs, dtype=tf.float32),
tf.cast(labels, dtype=tf.int32),
sequence_lengths)
print('Epoch:', str(epoch + 1), 'Batch:', iter_info[0], '/',
iter_info[1], 'Loss:', loss)
# save weights after each epoch
self.model.save_weights(FilePaths.fnWeight)
self.validate(loader)
def validate(self, loader):
self.load_weights(FilePaths.fnWeight)
# print('Validate NN')
loader.validation_set()
num_char_err = 0
num_char_total = 0
num_word_ok = 0
num_word_total = 0
while loader.has_next():
iter_info = loader.get_iterator_info()
print('Batch:', iter_info[0], '/', iter_info[1])
batch = loader.get_next()
pred = self.model(batch.imgs)
sequence_lengths = tf.cast(tf.fill([MyModel.batch_size],
MyModel.max_text_len),
dtype=tf.int32)
pred = tf.nn.ctc_beam_search_decoder(pred,
sequence_lengths,
beam_width=50)
recognized = decoder_output_to_text(pred, MyModel.batch_size)
print('Ground truth -> Recognized')
for i in range(len(recognized)):
num_word_ok += 1 if batch.gtTexts[i] == recognized[i] else 0
num_word_total += 1
dist = editdistance.eval(recognized[i], batch.gtTexts[i])
num_char_err += dist
num_char_total += len(batch.gtTexts[i])
print('[OK]' if dist == 0 else '[ERR:%d]' % dist,
'"' + batch.gtTexts[i] + '"', '->',
'"' + recognized[i] + '"')
# print validation result
char_error_rate = num_char_err / num_char_total
word_accuracy = num_word_ok / num_word_total
print('Character error rate: %f%%. Word accuracy: %f%%.' %
(char_error_rate * 100.0, word_accuracy * 100.0))
return char_error_rate
def infer(self):
img = cv2.imread(FilePaths.fnInfer, cv2.IMREAD_GRAYSCALE)
img = sample_preprocessor.pre_process(
img, (MyModel.img_width, MyModel.img_height))
self.load_weights(FilePaths.fnWeight)
pred = self.model(tf.expand_dims(img, axis=0))
sequence_lengths = tf.cast(tf.fill([1], MyModel.max_text_len),
dtype=tf.int32)
pred = tf.nn.ctc_beam_search_decoder(pred,
sequence_lengths,
beam_width=50)
# tf.print(pred[0][0])
pred = decoder_output_to_text(pred, 1)
print(pred)
def load_weights(self, file_path):
path = Path(file_path)
if path.is_file():
self.model.load_weights(filepath=file_path)
print('Note > Previous Weights not available')
model.compile(optimizer=ko.Adam(lr=0.0001), loss=[logits_loss, value_loss])
sdae.compile(optimizer=ko.Adam(lr=0.001), loss='mse')
for i in range(len(pairs)):
environments.append(FXEnv(pairs[i]['pair'], spread=pairs[i]['spread']))
agents.append(
MyAgent(model,
sdae,
state_size,
action_size,
environments[i],
nm=str(i)))
training = True
if training:
for i in range(800000):
for agent in agents:
agent.train(i)
else:
model.load_weights('../../play/weights/model')
sdae.load_weights('../../play/weights/sdae')
env = environments[2]
obs = env.reset()
for i in range(1000):
obs = sdae.encode(obs.reshape(1, 60, 12))
action, _ = model.action_value(obs)
obs, re, done, _ = env.step(action)
env.render()
fy = 720.0
cx = 608.0
cy = 180.0
# I've downsampled the images, so:
fx /= 1.725
fy /= 1.67
cx /= 1.725
cy /= 1.67
model = MyModel(fx, fy, cx, cy)
model.build((None, 224, 720, 6))
model.summary()
if restore_path:
model.load_weights(restore_path)
train_loss = tf.keras.metrics.Mean(name='train_loss')
optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3)
# optimizer = tf.keras.optimizers.Adam(learning_rate=1e-4)
current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
log_dir = 'logs/' + current_time
summary_writer = tf.summary.create_file_writer(log_dir)
# save_path = 'checkpoints\\run_' + current_time
save_path = 'checkpoints/mymodel'
@tf.function
from model import MyModel
from picamera.array import PiRGBArray
from picamera import PiCamera
import time
import cv2
import numpy as np
MAX = 32767 # Joystick
CROP = range(60, 160)
SIZE = (320, 160)
RESIZE = (200, 66)
model = MyModel()
model.load_weights('train/model/model.h5')
model._make_predict_function()
camera = PiCamera()
camera.resolution = SIZE
camera.framerate = 10
camera.rotation = 180
rawCapture = PiRGBArray(camera, size=SIZE)
def process(image):
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
return cv2.resize(image[CROP], RESIZE)
if __name__ == '__main__':
def train_model(ds_name, encoder, paths):
"""The main function for executing network training. It loads the specified
dataset iterator, saliency model, and helper classes. Training is then
performed in a new session by iterating over all batches for a number of
epochs. After validation on an independent set, the model is saved and
the training history is updated.
Args:
ds_name (str): Denotes the dataset to be used during training.
paths (dict, str): A dictionary with all path elements.
"""
w_filename_template = "/%s_%s_%s_weights.h5" # [encoder]_[ds_name]_[loss_fn_name]_weights.h5
(train_ds, n_train), (val_ds, n_val) = data.load_train_dataset(ds_name, paths["data"])
print(">> Preparing model with encoder %s..." % encoder)
model = MyModel(encoder, ds_name, "train")
if ds_name != "salicon":
salicon_weights = paths["weights"] + w_filename_template % (encoder, "salicon", loss_fn_name)
if os.path.exists(salicon_weights):
print("Salicon weights are loaded!\n %s"%salicon_weights)
else:
download.download_pretrained_weights(paths["weights"], encoder, "salicon", loss_fn_name)
model.load_weights(salicon_weights)
del salicon_weights
model.summary()
n_epochs = config.PARAMS["n_epochs"]
# Preparing
loss_fn = globals().get(loss_fn_name, None)
optimizer = tf.keras.optimizers.Adam(config.PARAMS["learning_rate"])
train_metric = tf.keras.metrics.Mean(name="train_loss")
val_metric = tf.keras.metrics.Mean(name="val_loss")
ckpts_path = paths["ckpts"] + "%s/%s/%s/" % (encoder, ds_name, loss_fn_name)
ckpt = tf.train.Checkpoint(net=model, train_metric=train_metric, val_metric=val_metric)
ckpt_manager = tf.train.CheckpointManager(ckpt, ckpts_path, max_to_keep=n_epochs)
start_epoch = 0
# if a checkpoint exists, restore the latest checkpoint.
if ckpt_manager.latest_checkpoint:
ckpt.restore(ckpt_manager.latest_checkpoint).assert_consumed()
start_epoch = int(ckpt_manager.latest_checkpoint.split('-')[-1])
print ('Checkpoint restored:\n{}'.format(ckpt_manager.latest_checkpoint))
train_metric.reset_states()
val_metric.reset_states()
print("\n>> Start training model on %s..." % ds_name.upper())
print(("Training details:" +
"\n{0:<4}Number of epochs: {n_epochs:d}" +
"\n{0:<4}Batch size: {batch_size:d}" +
"\n{0:<4}Learning rate: {learning_rate:.1e}" +
"\n{0:<4}Loss function: {1}").format(" ", loss_fn_name, **config.PARAMS))
print("_" * 65)
if ds_name == "salicon" and start_epoch < 2:
model.freeze_unfreeze_encoder_trained_layers(True)
for epoch in range(start_epoch, n_epochs):
if ds_name == "salicon" and epoch == 2:
model.freeze_unfreeze_encoder_trained_layers(False)
train_progbar = Progbar(n_train, stateful_metrics=["train_loss"])
for train_x, train_y_true, train_ori_sizes, train_filenames in train_ds:
train_y_pred, train_loss = train_step(train_x, train_y_true, model, loss_fn, optimizer)
train_metric(train_loss)
train_progbar.add(train_x.shape[0], [("train_loss", train_metric.result())])
val_progbar = Progbar(n_val, stateful_metrics=["val_loss"])
for val_x, val_y_true, val_ori_sizes, val_filenames in val_ds:
val_y_pred, val_loss = val_step(val_x, val_y_true, model, loss_fn)
val_metric(val_loss)
val_progbar.add(val_x.shape[0], [("val_loss", val_metric.result())])
train_metrics_results = _print_metrics({"train_loss": train_metric})
val_metrics_results = _print_metrics({"val_loss": val_metric})
print('Epoch {} - {} - {}'.format(epoch+1, train_metrics_results, val_metrics_results))
ckpt_manager.save()
# Reset the metrics for the next epoch
train_metric.reset_states()
val_metric.reset_states()
# Picking best result
print(">> Picking best result")
min_val_loss = None
for i, checkpoint in enumerate(ckpt_manager.checkpoints):
ckpt.restore(checkpoint).assert_consumed()
train_metrics_results = _print_metrics({"train_loss": train_metric})
val_metrics_results = _print_metrics({"val_loss": val_metric})
print('Epoch {} - {} - {}'.format(i+1, train_metrics_results, val_metrics_results))
val_loss_result = val_metric.result()
if min_val_loss is None or min_val_loss > val_loss_result:
min_train_loss = train_metric.result()
min_val_loss = val_loss_result
min_index = i
ckpt.restore(ckpt_manager.checkpoints[min_index])
print("best result picked -> epoch: {0} - train_{1}: {2} - val_{1}: {3}".format(min_index + 1, loss_fn_name,
('%.4f' if min_train_loss > 1e-3 else '%.4e') % min_train_loss,
('%.4f' if min_val_loss > 1e-3 else '%.4e') % min_val_loss))
# Saving model's weights
print(">> Saving model's weights")
dest_path = paths["weights"] + w_filename_template % (encoder, ds_name, loss_fn_name)
if min_index < 2:
model.freeze_unfreeze_encoder_trained_layers(False)
model.save_weights(dest_path)
print("weights are saved to:\n%s" % dest_path)
def find_n_high(ds_name, encoder, paths, n, metric, negate=False):
"""The main function for executing network training. It loads the specified
dataset iterator, saliency model, and helper classes. Training is then
performed in a new session by iterating over all batches for a number of
epochs. After validation on an independent set, the model is saved and
the training history is updated.
Args:
ds_name (str): Denotes the dataset to be used during training.
paths (dict, str): A dictionary with all path elements.
"""
w_filename_template = "/%s_%s_%s_weights.h5" # [encoder]_[ds_name]_[loss_fn_name]_weights.h5
(eval_ds, n_eval) = data.load_eval_dataset(ds_name, paths["data"])
print(">> Preparing model with encoder %s..." % encoder)
model = MyModel(encoder, ds_name, "train")
if "trained_weights" in paths:
if os.path.exists(paths["trained_weights"]):
weights_path = paths["trained_weights"]
else:
raise ValueError("could not find the specified weights file.\n specified weights: %s"%paths["trained_weights"])
else:
weights_path = paths["weights"] + w_filename_template % (encoder, ds_name, loss_fn_name)
if os.path.exists(weights_path):
print("Weights are loaded!\n %s"%weights_path)
else:
download.download_pretrained_weights(paths["weights"], encoder, "salicon", loss_fn_name)
model.load_weights(weights_path)
del weights_path
model.summary()
# Preparing
print("\n>> Start finding %d %s results for model on %s..." % (n, "worst" if negate else "best",ds_name.upper()))
print(("Evaluation details:" +
"\n{0:<4}Metric: {1}").format(" ", metric))
print("_" * 65)
eval_progbar = Progbar(n_eval)
min_heap = []
count = 0
sign = -1 if negate else 1
for eval_x, eval_fixs, eval_y_true, eval_ori_sizes, eval_filenames in eval_ds:
eval_y_pred = test_step(eval_x, model)
for pred, y_true, fixs, filename, ori_size in zip(eval_y_pred, eval_fixs, eval_y_true, eval_filenames.numpy(), eval_ori_sizes):
pred = tf.expand_dims(data.postprocess_saliency_map(pred, ori_size), axis=0)
fixs = tf.expand_dims(fixs, axis=0)
y_true = tf.expand_dims(y_true, axis=0)
score = _calc_metrics([metric], y_true, fixs, pred)[metric].numpy() * sign
if count < n:
count+=1
heapq.heappush(min_heap, (score, filename.decode("utf-8")))
else:
heapq.heappushpop(min_heap, (score, filename.decode("utf-8")))
eval_progbar.add(eval_x.shape[0])
min_heap.sort(reverse=True)
for s, n in min_heap:
print(s, n)
def eval_results(ds_name, encoder, paths):
"""The main function for executing network training. It loads the specified
dataset iterator, saliency model, and helper classes. Training is then
performed in a new session by iterating over all batches for a number of
epochs. After validation on an independent set, the model is saved and
the training history is updated.
Args:
ds_name (str): Denotes the dataset to be used during training.
paths (dict, str): A dictionary with all path elements.
"""
w_filename_template = "/%s_%s_%s_weights.h5" # [encoder]_[ds_name]_[loss_fn_name]_weights.h5
(eval_ds, n_eval) = data.load_eval_dataset(ds_name, paths["data"])
print(">> Preparing model with encoder %s..." % encoder)
model = MyModel(encoder, ds_name, "train")
if "trained_weights" in paths:
if os.path.exists(paths["trained_weights"]):
weights_path = paths["trained_weights"]
else:
raise ValueError("could not find the specified weights file.\n specified weights: %s"%paths["trained_weights"])
else:
weights_path = paths["weights"] + w_filename_template % (encoder, ds_name, loss_fn_name)
if os.path.exists(weights_path):
print("Weights are loaded!\n %s"%weights_path)
else:
download.download_pretrained_weights(paths["weights"], encoder, "salicon", loss_fn_name)
model.load_weights(weights_path)
del weights_path
model.summary()
# Preparing
metrics = config.PARAMS["metrics"]
print("\n>> Start evaluating model on %s..." % ds_name.upper())
print(("Evaluation details:" +
"\n{0:<4}Metrics: {2}").format(" ", loss_fn_name, ", ".join(metrics), **config.PARAMS))
print("_" * 65)
eval_progbar = Progbar(n_eval)
categorical = config.SPECS[ds_name].get("categorical", False)
cat_metrics = {}
for eval_x, eval_fixs, eval_y_true, eval_ori_sizes, eval_filenames in eval_ds:
eval_y_pred = test_step(eval_x, model)
for pred, y_true, fixs, filename, ori_size in zip(eval_y_pred, eval_fixs, eval_y_true, eval_filenames.numpy(), eval_ori_sizes):
pred = tf.expand_dims(data.postprocess_saliency_map(pred, ori_size), axis=0)
fixs = tf.expand_dims(fixs, axis=0)
y_true = tf.expand_dims(y_true, axis=0)
met_vals = _calc_metrics(metrics, y_true, fixs, pred)
if categorical:
cat = "/".join(filename.decode("utf-8").split("/")[:-1])
if not cat in cat_metrics:
cat_metrics[cat] = {}
for name in metrics:
cat_metrics[cat][name] = {"sum":0, "count": 0}
for name, value in met_vals.items():
cat_metrics[cat][name]["sum"] += value
cat_metrics[cat][name]["count"] += 1
eval_progbar.add(eval_x.shape[0], met_vals.items())
for cat, cat_met in cat_metrics.items():
to_print = []
for name, value in cat_met.items():
_mean = value["sum"]/value["count"]
to_print.append("{}: {}".format(name, ('%.4f' if _mean > 1e-3 else '%.4e') % _mean))
print('Results ({}): {}'.format(cat, " - ".join(to_print)))
