def train_and_predict(data_path, data_filename, batch_size, n_epoch): """ Create a model, load the data, and train it. """ """ Step 1: Load the data """ hdf5_filename = os.path.join(data_path, data_filename) print("-" * 30) print("Loading the data from HDF5 file ...") print("-" * 30) imgs_train, msks_train, imgs_validation, msks_validation = \ load_data(hdf5_filename) print("-" * 30) print("Creating and compiling model ...") print("-" * 30) """ Step 2: Define the model """ model = load_model(imgs_train.shape, msks_train.shape) model_filename, model_callbacks = get_callbacks() """ Step 3: Train the model on the data """ print("-" * 30) print("Fitting model with training data ...") print("-" * 30) history = model.fit(imgs_train, msks_train, batch_size=batch_size, epochs=n_epoch, validation_data=(imgs_validation, msks_validation), verbose=1, shuffle="batch", callbacks=model_callbacks) # Append training log # with open("training.log","a+") as fp: # fp.write("{}: {}\n".format(datetime.datetime.now(), # history.history["val_dice_coef"])) """ Step 4: Evaluate the best model """ print("-" * 30) print("Loading the best trained model ...") print("-" * 30) model = evaluate_model(model_filename, imgs_validation, msks_validation) """ Step 5: Save the best model for inference """ print("-" * 30) print("Saving the model for inference ...") print("-" * 30) save_inference_model(model, imgs_train.shape, msks_train.shape)
def set_up_model(model_name, input_shape, model_dir): if not os.path.isdir(model_dir): logger.info("Create directory {}".format(model_dir)) os.makedirs(model_dir) model = get_model(input_shape, model_name, nclass=2) # callbacks callbacks = get_callbacks(os.path.join(model_dir, model_name)) return model, callbacks
def train_model(train_filelist, config, classes, rare_classes): batch_size = config['train_params']['batch_size'] different_classes_per_batch = config['train_params']['different_classes_per_batch'] nb_epoch = config['train_params']['num_epochs'] steps_per_epoch = config['train_params']['steps_per_epoch'] img_size = config['img']['img_size'] model = prepare_model(config) callbacks = get_callbacks(config) model.fit_generator(data_generator(train_filelist, classes, rare_classes, img_size, batch_size, different_classes_per_batch), steps_per_epoch=steps_per_epoch, epochs=nb_epoch, callbacks=callbacks) return model
def train_model(): arguments = parcero() (x_train, y_train), (x_test, y_test) = dataprep.get_and_pad_imdb_dataset(vocab_size=arguments.vocab_size, maxlen=arguments.max_seq_len) imdb_word_index = dataprep.get_imdb_word_index() max_index_value = max(imdb_word_index.values()) classifier = model.get_model(vocab_size=max_index_value) callbacks = model.get_callbacks(arguments.job_dir) history = classifier.fit(x_train, y_train, epochs=arguments.epochs, batch_size=arguments.batch_size, validation_data=(x_test, y_test), validation_steps=20, callbacks=callbacks)
def train_and_predict(data_path, data_filename, batch_size, n_epoch): """ Create a model, load the data, and train it. """ """ Step 1: Load the data """ hdf5_filename = os.path.join(data_path, data_filename) print("-" * 30) print("Loading the data from HDF5 file ...") print("-" * 30) imgs_train, msks_train, imgs_validation, msks_validation = \ load_data(hdf5_filename, args.batch_size, [args.crop_dim, args.crop_dim]) print("-" * 30) print("Creating and compiling model ...") print("-" * 30) """ Step 2: Define the model """ model = load_model(imgs_train.shape, msks_train.shape) model_filename, model_callbacks = get_callbacks() """ Step 3: Train the model on the data """ print("-" * 30) print("Fitting model with training data ...") print("-" * 30) model.fit(imgs_train, msks_train, batch_size=batch_size, epochs=n_epoch, validation_data=(imgs_validation, msks_validation), verbose=1, shuffle="batch", callbacks=model_callbacks) """ Step 4: Evaluate the best model """ print("-" * 30) print("Loading the best trained model ...") print("-" * 30) model = evaluate_model(model_filename, imgs_validation, msks_validation)
def train(): generator_train = data_generator(train_tfrecord_dir, "train") generator_test = data_generator(test_tfrecord_dir, "test") images, masks = generator_train.__next__() # save_img(images, masks) input_shape = (512, 512, 1) model = uent2d_model(input_shape=input_shape, loss_function=keras.losses.binary_crossentropy) # 打印模型结构 model.summary() # # 保存模型图 # from keras.utils import plot_model # plot_model(model, to_file="model.png") tensorboard = keras.callbacks.TensorBoard( log_dir='./logs', # log 目录 histogram_freq=0, # 按照何等频率(epoch)来计算直方图,0为不计算 batch_size=5, # 用多大量的数据计算直方图 write_graph=True, # 是否存储网络结构图 write_grads=True, # 是否可视化梯度直方图 write_images=True, # 是否可视化参数 embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None) if not os.path.exists("./ckpt"): os.mkdir("./ckpt") filepath = "./ckpt/{epoch:03d}-{val_loss:.4f}.h5" callbacks = get_callbacks(filepath) callbacks.append(tensorboard) model.fit_generator(generator_train, validation_data=generator_test, steps_per_epoch=steps_per_epoch, callbacks=callbacks, epochs=epochs, verbose=1, validation_steps=1) model.save("./ckpt/save_model.h5") return None
TEST_SAMPLES = len(all_test_label) num_steps_train = tf.math.ceil(float(TRAIN_SAMPLES) / BATCH_SIZE) num_steps_val = tf.math.ceil(float(VALID_SAMPLES) / BATCH_SIZE) num_steps_test = tf.math.ceil(float(TEST_SAMPLES) / BATCH_SIZE) # outs = flor(input_size, 143, 0.001) outs = minh_model(input_size, 143) inputs, outputs, optimizer = outs model = Model(inputs=inputs, outputs=outputs) model.compile(optimizer=optimizer, loss=ctc_loss_lambda_func) print(model.summary()) checkpoint = './models/checkpoint_1.hdf5' callback = get_callbacks(checkpoint) start_time = datetime.datetime.now() history = model.fit(train_ds, steps_per_epoch=num_steps_train, epochs=20, validation_data=valid_ds, callback=callback, validation_steps=num_steps_val) total_time = datetime.datetime.now() - start_time loss = history.history['loss'] val_loss = history.history['val_loss']
parser.add_argument("--train", action="store_true", default=False) parser.add_argument("--test", action="store_true", default=False) parser.add_argument("--path", type=str, required=False) # path to test data args = parser.parse_args() # ------------- TRAIN ------------- if args.train: train_ds, num_steps_train, _ = build_dataset('train', cache=True, augment=True) test_ds, num_steps_val, _ = build_dataset('test', training=False) model = mb.build_model(input_size=INPUT_SIZE, d_model=vocab_size + 1, learning_rate=0.001) callbacks = mb.get_callbacks('weights_1.hdf5', 'val_loss', 1) batch_stats_callback = mb.CollectBatchStats() start_time = datetime.datetime.now() h = model.fit(train_ds, steps_per_epoch=num_steps_train, epochs=100, validation_data=test_ds, validation_steps=num_steps_val, callbacks=callbacks) total_time = datetime.datetime.now() - start_time loss = h.history['loss'] val_loss = h.history['val_loss'] min_val_loss = min(val_loss)