print("train step start\n") for epoch in range(load_weights_from_epoch + 1, EPOCHS): start_time = time.time() for step, batch_data in enumerate(train_data): #print(tf.size(batch_data),"\n") images, labels = ReadDataset().read(batch_data)#images是一个[batch_size, 300,300,3],lable是[batch_size,20,5]的tensor #print(tf.shape(labels)) train_step(batch_images=images, batch_labels=labels) time_per_step = (time.time() - start_time) / (step + 1) print("Epoch: {}/{}, step: {}/{}, {:.2f}s/step, loss: {:.5f}, " "cls loss: {:.5f}, reg loss: {:.5f}".format(epoch, EPOCHS, step, tf.math.ceil(train_count / BATCH_SIZE), time_per_step, loss_metric.result(), cls_loss_metric.result(), reg_loss_metric.result())) loss_metric.reset_states() cls_loss_metric.reset_states() reg_loss_metric.reset_states() if epoch % save_frequency == 0: ssd.save_weights(filepath=save_model_dir+"epoch-{}".format(epoch), save_format="tf") if test_images_during_training: visualize_training_results(pictures=test_images_dir_list, model=ssd, epoch=epoch) ssd.save_weights(filepath=save_model_dir+"saved_model", save_format="tf")
def main(): dataset = TFDataset() train_data, train_count = dataset.generate_datatset() model = SSD() print_model_summary(model) if load_weights_from_epoch >= 0: model.load_weights(filepath=save_model_dir + "epoch-{}".format(load_weights_from_epoch)) print("成功从epoch-{}加载模型权重!".format(load_weights_from_epoch)) loss_fn = MultiBoxLoss(num_classes=NUM_CLASSES, overlap_thresh=0.5, neg_pos=3) # optimizer lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay( initial_learning_rate=1e-3, decay_steps=20000, decay_rate=0.96) optimizer = tf.optimizers.Adam(learning_rate=lr_schedule) loss_metric = tf.metrics.Mean() cls_loss_metric = tf.metrics.Mean() reg_loss_metric = tf.metrics.Mean() for epoch in range(load_weights_from_epoch + 1, EPOCHS): start_time = time.time() for step, batch_data in enumerate(train_data): images, labels = ReadDataset().read(batch_data) with tf.GradientTape() as tape: predictions = model(images, training=True) loss_l, loss_c = loss_fn(y_true=labels, y_pred=predictions) total_loss = loss_l + loss_c gradients = tape.gradient(total_loss, model.trainable_variables) optimizer.apply_gradients( grads_and_vars=zip(gradients, model.trainable_variables)) loss_metric.update_state(values=total_loss) cls_loss_metric.update_state(values=loss_c) reg_loss_metric.update_state(values=loss_l) time_per_step = (time.time() - start_time) / (step + 1) print( "Epoch: {}/{}, step: {}/{}, speed: {:.2f}s/step, loss: {:.10f}, " "cls loss: {:.10f}, reg loss: {:.10f}".format( epoch, EPOCHS, step, tf.math.ceil(train_count / BATCH_SIZE), time_per_step, loss_metric.result(), cls_loss_metric.result(), reg_loss_metric.result())) loss_metric.reset_states() cls_loss_metric.reset_states() reg_loss_metric.reset_states() if epoch % save_frequency == 0: model.save_weights(filepath=save_model_dir + "epoch-{}".format(epoch), save_format="tf") if test_images_during_training: visualize_training_results(pictures=test_images_dir_list, model=model, epoch=epoch) model.save_weights(filepath=save_model_dir + "epoch-{}".format(EPOCHS), save_format="tf")
loss = yolo_loss(y_true=label_batch, y_pred=yolo_output) gradients = tape.gradient(loss, net.trainable_variables) optimizer.apply_gradients( grads_and_vars=zip(gradients, net.trainable_variables)) loss_metric.update_state(values=loss) for epoch in range(load_weights_from_epoch + 1, EPOCHS): step = 0 for dataset_batch in train_dataset: step += 1 images, boxes = parse_dataset_batch(dataset=dataset_batch) labels = generate_label_batch(true_boxes=boxes) train_step(image_batch=process_image_filenames(images), label_batch=labels) print("Epoch: {}/{}, step: {}/{}, loss: {:.5f}".format( epoch, EPOCHS, step, tf.math.ceil(train_count / BATCH_SIZE), loss_metric.result())) loss_metric.reset_states() if epoch % save_frequency == 0: net.save_weights(filepath=save_model_dir + "epoch-{}".format(epoch), save_format='tf') if test_images_during_training: visualize_training_results(pictures=test_images, model=net, epoch=epoch) net.save_weights(filepath=save_model_dir + "saved_model", save_format='tf')
gradients = tape.gradient(target=loss_value, sources=centernet.trainable_variables) optimizer.apply_gradients( grads_and_vars=zip(gradients, centernet.trainable_variables)) loss_metric.update_state(values=loss_value) for epoch in range(load_weights_from_epoch + 1, Config.epochs): for step, batch_data in enumerate(train_data): step_start_time = time.time() images, labels = data_loader.read_batch_data(batch_data) train_step(images, labels) step_end_time = time.time() print("Epoch: {}/{}, step: {}/{}, loss: {}, time_cost: {:.3f}s". format(epoch, Config.epochs, step, steps_per_epoch, loss_metric.result(), step_end_time - step_start_time)) loss_metric.reset_states() if epoch % Config.save_frequency == 0: centernet.save_weights(filepath=Config.save_model_dir + "epoch-{}".format(epoch), save_format="tf") if Config.test_images_during_training: visualize_training_results(pictures=Config.test_images_dir_list, model=centernet, epoch=epoch) centernet.save_weights(filepath=Config.save_model_dir + "saved_model", save_format="tf")