def predict(title, description): # Load model and weights model = MyModel() model.load_weights(model_name) # Clean the description des = clean_data_for_overview(description) # Predict genre = model.predict(np.array([des])) # Prepare the output res = r'''{ "title": "%s", "description": "%s", "genre": "%s" }''' % (title, description, genre[0]) print(res)
def test_model(ds_name, encoder, paths, categorical=False): """The main function for executing network testing. It loads the specified dataset iterator and optimized saliency model. By default, when no model checkpoint is found locally, the pretrained weights will be downloaded. Args: ds_name (str): Denotes the dataset that was used during training. encoder (str): the name of the encoder want to be used to predict. paths (dict, str): A dictionary with all path elements. """ w_filename_template = "/%s_%s_%s_weights.h5" # [encoder]_[ds_name]_weights.h5 (test_ds, n_test) = data.load_test_dataset(ds_name, paths["data"], categorical) print(">> Preparing model with encoder %s..." % encoder) model = MyModel(encoder, ds_name, "test") weights_path = paths["weights"] + w_filename_template % (encoder, ds_name, loss_fn_name) if os.path.exists(weights_path): print("Weights are loaded!\n %s"%weights_path) else: download.download_pretrained_weights(paths["weights"], encoder, ds_name, loss_fn_name) model.load_weights(weights_path) del weights_path print(">> Start predicting using model trained on %s..." % ds_name.upper()) results_path = paths["results"] + "%s/%s/%s/" % (ds_name, encoder, loss_fn_name) # Preparing progbar test_progbar = Progbar(n_test) for test_images, test_ori_sizes, test_filenames in test_ds: pred = test_step(test_images, model) for pred, filename, ori_size in zip(pred, test_filenames.numpy(), test_ori_sizes): img = data.postprocess_saliency_map(pred, ori_size, as_image=True) tf.io.write_file(results_path + filename.decode("utf-8"), img) test_progbar.add(test_images.shape[0])
# lr ExponentialDecay lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay( config['learning_rate'], decay_steps=config['decay_steps'], decay_rate=config['decay_rate']) model.compile( optimizer=tf.keras.optimizers.Adam(learning_rate=lr_schedule), loss=Loss(), metrics=[PSNR(), SSIM()], ) # resume checkpoint if config['resume']: model.build((None, None, None, 3)) model.load_weights(checkpoint_path) # save the best model checkpoint model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint( filepath=checkpoint_path, save_weights_only=True, monitor='val_loss', mode='min', save_best_only=True) # tensorboard visulization tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1) # model fit model.fit(trainset,
# COLORMAP = [[0, 0, 255], [0, 255, 0]] # cm = np.array(COLORMAP).astype(np.uint8) # pred = np.argmax(np.array(pred), axis=2) # pred_val = cm[pred] # cv2.imwrite(os.path.join("data",filename.split("/")[-1]), pred_val) # print(os.path.join("data",filename.split("/")[-1])+"finished") test_dataset = tf.data.Dataset.from_generator( test_generator, tf.float32, tf.TensorShape([None, None, None])) test_dataset = test_dataset.batch(5) #model = DeepLabV3Plus(image_shape[0], image_shape[1], nclasses=4) model = MyModel(4) model.load_weights(weight_path + 'fcn_20191021.ckpt') test_list_dir = os.listdir(test_dir) test_list_dir.sort() test_filenames = [test_dir + filename for filename in test_list_dir] for filename in test_filenames: image = scipy.misc.imresize(scipy.misc.imread(filename), image_shape) # image dim=[h, w, channel] #image = image/255 #image_yuv = cv2.cvtColor(image, cv2.COLOR_RGB2YUV) #image_yuv[:, :, 0] = cv2.equalizeHist(image_yuv[:, :, 0]) #image = cv2.cvtColor(image_yuv, cv2.COLOR_YUV2RGB) image = image[np.newaxis, :, :, :].astype("float32")
class ModelHelper: def __init__(self): self.model = MyModel() # change the learning rate rate to start training from scratch self.optimizer = tf.keras.optimizers.RMSprop(learning_rate=0.00001) @tf.function def train_step(self, images, labels, seq_len): with tf.GradientTape() as tape: predictions = self.model(images) loss = tf.math.reduce_mean( tf.nn.ctc_loss(labels=labels, logits=predictions, logit_length=seq_len, label_length=None, blank_index=-1)) gradients = tape.gradient(loss, self.model.trainable_variables) self.optimizer.apply_gradients( zip(gradients, self.model.trainable_variables)) return loss def train(self, loader): # load previously trained weights self.load_weights(FilePaths.fnWeight) # define the number of epochs to run epochs = 1 for epoch in range(epochs): loader.train_set() while loader.has_next(): iter_info = loader.get_iterator_info() batch = loader.get_next() labels = to_sparse(batch.gtTexts) labels = tf.SparseTensor(labels[0], labels[1], labels[2]) sequence_lengths = tf.cast(tf.fill([MyModel.batch_size], MyModel.max_text_len), dtype=tf.int32) loss = self.train_step(tf.cast(batch.imgs, dtype=tf.float32), tf.cast(labels, dtype=tf.int32), sequence_lengths) print('Epoch:', str(epoch + 1), 'Batch:', iter_info[0], '/', iter_info[1], 'Loss:', loss) # save weights after each epoch self.model.save_weights(FilePaths.fnWeight) self.validate(loader) def validate(self, loader): self.load_weights(FilePaths.fnWeight) # print('Validate NN') loader.validation_set() num_char_err = 0 num_char_total = 0 num_word_ok = 0 num_word_total = 0 while loader.has_next(): iter_info = loader.get_iterator_info() print('Batch:', iter_info[0], '/', iter_info[1]) batch = loader.get_next() pred = self.model(batch.imgs) sequence_lengths = tf.cast(tf.fill([MyModel.batch_size], MyModel.max_text_len), dtype=tf.int32) pred = tf.nn.ctc_beam_search_decoder(pred, sequence_lengths, beam_width=50) recognized = decoder_output_to_text(pred, MyModel.batch_size) print('Ground truth -> Recognized') for i in range(len(recognized)): num_word_ok += 1 if batch.gtTexts[i] == recognized[i] else 0 num_word_total += 1 dist = editdistance.eval(recognized[i], batch.gtTexts[i]) num_char_err += dist num_char_total += len(batch.gtTexts[i]) print('[OK]' if dist == 0 else '[ERR:%d]' % dist, '"' + batch.gtTexts[i] + '"', '->', '"' + recognized[i] + '"') # print validation result char_error_rate = num_char_err / num_char_total word_accuracy = num_word_ok / num_word_total print('Character error rate: %f%%. Word accuracy: %f%%.' % (char_error_rate * 100.0, word_accuracy * 100.0)) return char_error_rate def infer(self): img = cv2.imread(FilePaths.fnInfer, cv2.IMREAD_GRAYSCALE) img = sample_preprocessor.pre_process( img, (MyModel.img_width, MyModel.img_height)) self.load_weights(FilePaths.fnWeight) pred = self.model(tf.expand_dims(img, axis=0)) sequence_lengths = tf.cast(tf.fill([1], MyModel.max_text_len), dtype=tf.int32) pred = tf.nn.ctc_beam_search_decoder(pred, sequence_lengths, beam_width=50) # tf.print(pred[0][0]) pred = decoder_output_to_text(pred, 1) print(pred) def load_weights(self, file_path): path = Path(file_path) if path.is_file(): self.model.load_weights(filepath=file_path) print('Note > Previous Weights not available')
model.compile(optimizer=ko.Adam(lr=0.0001), loss=[logits_loss, value_loss]) sdae.compile(optimizer=ko.Adam(lr=0.001), loss='mse') for i in range(len(pairs)): environments.append(FXEnv(pairs[i]['pair'], spread=pairs[i]['spread'])) agents.append( MyAgent(model, sdae, state_size, action_size, environments[i], nm=str(i))) training = True if training: for i in range(800000): for agent in agents: agent.train(i) else: model.load_weights('../../play/weights/model') sdae.load_weights('../../play/weights/sdae') env = environments[2] obs = env.reset() for i in range(1000): obs = sdae.encode(obs.reshape(1, 60, 12)) action, _ = model.action_value(obs) obs, re, done, _ = env.step(action) env.render()
fy = 720.0 cx = 608.0 cy = 180.0 # I've downsampled the images, so: fx /= 1.725 fy /= 1.67 cx /= 1.725 cy /= 1.67 model = MyModel(fx, fy, cx, cy) model.build((None, 224, 720, 6)) model.summary() if restore_path: model.load_weights(restore_path) train_loss = tf.keras.metrics.Mean(name='train_loss') optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3) # optimizer = tf.keras.optimizers.Adam(learning_rate=1e-4) current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") log_dir = 'logs/' + current_time summary_writer = tf.summary.create_file_writer(log_dir) # save_path = 'checkpoints\\run_' + current_time save_path = 'checkpoints/mymodel' @tf.function
from model import MyModel from picamera.array import PiRGBArray from picamera import PiCamera import time import cv2 import numpy as np MAX = 32767 # Joystick CROP = range(60, 160) SIZE = (320, 160) RESIZE = (200, 66) model = MyModel() model.load_weights('train/model/model.h5') model._make_predict_function() camera = PiCamera() camera.resolution = SIZE camera.framerate = 10 camera.rotation = 180 rawCapture = PiRGBArray(camera, size=SIZE) def process(image): image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) return cv2.resize(image[CROP], RESIZE) if __name__ == '__main__':
def train_model(ds_name, encoder, paths): """The main function for executing network training. It loads the specified dataset iterator, saliency model, and helper classes. Training is then performed in a new session by iterating over all batches for a number of epochs. After validation on an independent set, the model is saved and the training history is updated. Args: ds_name (str): Denotes the dataset to be used during training. paths (dict, str): A dictionary with all path elements. """ w_filename_template = "/%s_%s_%s_weights.h5" # [encoder]_[ds_name]_[loss_fn_name]_weights.h5 (train_ds, n_train), (val_ds, n_val) = data.load_train_dataset(ds_name, paths["data"]) print(">> Preparing model with encoder %s..." % encoder) model = MyModel(encoder, ds_name, "train") if ds_name != "salicon": salicon_weights = paths["weights"] + w_filename_template % (encoder, "salicon", loss_fn_name) if os.path.exists(salicon_weights): print("Salicon weights are loaded!\n %s"%salicon_weights) else: download.download_pretrained_weights(paths["weights"], encoder, "salicon", loss_fn_name) model.load_weights(salicon_weights) del salicon_weights model.summary() n_epochs = config.PARAMS["n_epochs"] # Preparing loss_fn = globals().get(loss_fn_name, None) optimizer = tf.keras.optimizers.Adam(config.PARAMS["learning_rate"]) train_metric = tf.keras.metrics.Mean(name="train_loss") val_metric = tf.keras.metrics.Mean(name="val_loss") ckpts_path = paths["ckpts"] + "%s/%s/%s/" % (encoder, ds_name, loss_fn_name) ckpt = tf.train.Checkpoint(net=model, train_metric=train_metric, val_metric=val_metric) ckpt_manager = tf.train.CheckpointManager(ckpt, ckpts_path, max_to_keep=n_epochs) start_epoch = 0 # if a checkpoint exists, restore the latest checkpoint. if ckpt_manager.latest_checkpoint: ckpt.restore(ckpt_manager.latest_checkpoint).assert_consumed() start_epoch = int(ckpt_manager.latest_checkpoint.split('-')[-1]) print ('Checkpoint restored:\n{}'.format(ckpt_manager.latest_checkpoint)) train_metric.reset_states() val_metric.reset_states() print("\n>> Start training model on %s..." % ds_name.upper()) print(("Training details:" + "\n{0:<4}Number of epochs: {n_epochs:d}" + "\n{0:<4}Batch size: {batch_size:d}" + "\n{0:<4}Learning rate: {learning_rate:.1e}" + "\n{0:<4}Loss function: {1}").format(" ", loss_fn_name, **config.PARAMS)) print("_" * 65) if ds_name == "salicon" and start_epoch < 2: model.freeze_unfreeze_encoder_trained_layers(True) for epoch in range(start_epoch, n_epochs): if ds_name == "salicon" and epoch == 2: model.freeze_unfreeze_encoder_trained_layers(False) train_progbar = Progbar(n_train, stateful_metrics=["train_loss"]) for train_x, train_y_true, train_ori_sizes, train_filenames in train_ds: train_y_pred, train_loss = train_step(train_x, train_y_true, model, loss_fn, optimizer) train_metric(train_loss) train_progbar.add(train_x.shape[0], [("train_loss", train_metric.result())]) val_progbar = Progbar(n_val, stateful_metrics=["val_loss"]) for val_x, val_y_true, val_ori_sizes, val_filenames in val_ds: val_y_pred, val_loss = val_step(val_x, val_y_true, model, loss_fn) val_metric(val_loss) val_progbar.add(val_x.shape[0], [("val_loss", val_metric.result())]) train_metrics_results = _print_metrics({"train_loss": train_metric}) val_metrics_results = _print_metrics({"val_loss": val_metric}) print('Epoch {} - {} - {}'.format(epoch+1, train_metrics_results, val_metrics_results)) ckpt_manager.save() # Reset the metrics for the next epoch train_metric.reset_states() val_metric.reset_states() # Picking best result print(">> Picking best result") min_val_loss = None for i, checkpoint in enumerate(ckpt_manager.checkpoints): ckpt.restore(checkpoint).assert_consumed() train_metrics_results = _print_metrics({"train_loss": train_metric}) val_metrics_results = _print_metrics({"val_loss": val_metric}) print('Epoch {} - {} - {}'.format(i+1, train_metrics_results, val_metrics_results)) val_loss_result = val_metric.result() if min_val_loss is None or min_val_loss > val_loss_result: min_train_loss = train_metric.result() min_val_loss = val_loss_result min_index = i ckpt.restore(ckpt_manager.checkpoints[min_index]) print("best result picked -> epoch: {0} - train_{1}: {2} - val_{1}: {3}".format(min_index + 1, loss_fn_name, ('%.4f' if min_train_loss > 1e-3 else '%.4e') % min_train_loss, ('%.4f' if min_val_loss > 1e-3 else '%.4e') % min_val_loss)) # Saving model's weights print(">> Saving model's weights") dest_path = paths["weights"] + w_filename_template % (encoder, ds_name, loss_fn_name) if min_index < 2: model.freeze_unfreeze_encoder_trained_layers(False) model.save_weights(dest_path) print("weights are saved to:\n%s" % dest_path)
def find_n_high(ds_name, encoder, paths, n, metric, negate=False): """The main function for executing network training. It loads the specified dataset iterator, saliency model, and helper classes. Training is then performed in a new session by iterating over all batches for a number of epochs. After validation on an independent set, the model is saved and the training history is updated. Args: ds_name (str): Denotes the dataset to be used during training. paths (dict, str): A dictionary with all path elements. """ w_filename_template = "/%s_%s_%s_weights.h5" # [encoder]_[ds_name]_[loss_fn_name]_weights.h5 (eval_ds, n_eval) = data.load_eval_dataset(ds_name, paths["data"]) print(">> Preparing model with encoder %s..." % encoder) model = MyModel(encoder, ds_name, "train") if "trained_weights" in paths: if os.path.exists(paths["trained_weights"]): weights_path = paths["trained_weights"] else: raise ValueError("could not find the specified weights file.\n specified weights: %s"%paths["trained_weights"]) else: weights_path = paths["weights"] + w_filename_template % (encoder, ds_name, loss_fn_name) if os.path.exists(weights_path): print("Weights are loaded!\n %s"%weights_path) else: download.download_pretrained_weights(paths["weights"], encoder, "salicon", loss_fn_name) model.load_weights(weights_path) del weights_path model.summary() # Preparing print("\n>> Start finding %d %s results for model on %s..." % (n, "worst" if negate else "best",ds_name.upper())) print(("Evaluation details:" + "\n{0:<4}Metric: {1}").format(" ", metric)) print("_" * 65) eval_progbar = Progbar(n_eval) min_heap = [] count = 0 sign = -1 if negate else 1 for eval_x, eval_fixs, eval_y_true, eval_ori_sizes, eval_filenames in eval_ds: eval_y_pred = test_step(eval_x, model) for pred, y_true, fixs, filename, ori_size in zip(eval_y_pred, eval_fixs, eval_y_true, eval_filenames.numpy(), eval_ori_sizes): pred = tf.expand_dims(data.postprocess_saliency_map(pred, ori_size), axis=0) fixs = tf.expand_dims(fixs, axis=0) y_true = tf.expand_dims(y_true, axis=0) score = _calc_metrics([metric], y_true, fixs, pred)[metric].numpy() * sign if count < n: count+=1 heapq.heappush(min_heap, (score, filename.decode("utf-8"))) else: heapq.heappushpop(min_heap, (score, filename.decode("utf-8"))) eval_progbar.add(eval_x.shape[0]) min_heap.sort(reverse=True) for s, n in min_heap: print(s, n)
def eval_results(ds_name, encoder, paths): """The main function for executing network training. It loads the specified dataset iterator, saliency model, and helper classes. Training is then performed in a new session by iterating over all batches for a number of epochs. After validation on an independent set, the model is saved and the training history is updated. Args: ds_name (str): Denotes the dataset to be used during training. paths (dict, str): A dictionary with all path elements. """ w_filename_template = "/%s_%s_%s_weights.h5" # [encoder]_[ds_name]_[loss_fn_name]_weights.h5 (eval_ds, n_eval) = data.load_eval_dataset(ds_name, paths["data"]) print(">> Preparing model with encoder %s..." % encoder) model = MyModel(encoder, ds_name, "train") if "trained_weights" in paths: if os.path.exists(paths["trained_weights"]): weights_path = paths["trained_weights"] else: raise ValueError("could not find the specified weights file.\n specified weights: %s"%paths["trained_weights"]) else: weights_path = paths["weights"] + w_filename_template % (encoder, ds_name, loss_fn_name) if os.path.exists(weights_path): print("Weights are loaded!\n %s"%weights_path) else: download.download_pretrained_weights(paths["weights"], encoder, "salicon", loss_fn_name) model.load_weights(weights_path) del weights_path model.summary() # Preparing metrics = config.PARAMS["metrics"] print("\n>> Start evaluating model on %s..." % ds_name.upper()) print(("Evaluation details:" + "\n{0:<4}Metrics: {2}").format(" ", loss_fn_name, ", ".join(metrics), **config.PARAMS)) print("_" * 65) eval_progbar = Progbar(n_eval) categorical = config.SPECS[ds_name].get("categorical", False) cat_metrics = {} for eval_x, eval_fixs, eval_y_true, eval_ori_sizes, eval_filenames in eval_ds: eval_y_pred = test_step(eval_x, model) for pred, y_true, fixs, filename, ori_size in zip(eval_y_pred, eval_fixs, eval_y_true, eval_filenames.numpy(), eval_ori_sizes): pred = tf.expand_dims(data.postprocess_saliency_map(pred, ori_size), axis=0) fixs = tf.expand_dims(fixs, axis=0) y_true = tf.expand_dims(y_true, axis=0) met_vals = _calc_metrics(metrics, y_true, fixs, pred) if categorical: cat = "/".join(filename.decode("utf-8").split("/")[:-1]) if not cat in cat_metrics: cat_metrics[cat] = {} for name in metrics: cat_metrics[cat][name] = {"sum":0, "count": 0} for name, value in met_vals.items(): cat_metrics[cat][name]["sum"] += value cat_metrics[cat][name]["count"] += 1 eval_progbar.add(eval_x.shape[0], met_vals.items()) for cat, cat_met in cat_metrics.items(): to_print = [] for name, value in cat_met.items(): _mean = value["sum"]/value["count"] to_print.append("{}: {}".format(name, ('%.4f' if _mean > 1e-3 else '%.4e') % _mean)) print('Results ({}): {}'.format(cat, " - ".join(to_print)))