def train_models(file_path, model, datagen, epoches, batch_size, train_data,
valied_data):
X_train, y_train = train_data
X_valied, y_valied = valied_data
model.fit_generator(datagen.flow(X_train, y_train, batch_size=batch_size),
epochs=epoches,
steps_per_epoch=len(X_train) / batch_size,
validation_data=(X_valied, y_valied),
verbose=1,
callbacks=callbacks)
# @Desc :
import keras
from dataset import MyDataGenerator
from dataset import tokenizer
from dataset import train_data, dev_data, test_data
from models import model
import settings
# 创建数据集迭代器
train_generator = MyDataGenerator(train_data, tokenizer, settings.BATCH_SIZE)
dev_generator = MyDataGenerator(dev_data, tokenizer, settings.BATCH_SIZE)
test_generator = MyDataGenerator(test_data, tokenizer, settings.BATCH_SIZE)
# 设置checkpoint，自动保存模型
checkpoint = keras.callbacks.ModelCheckpoint(
settings.BEST_WEIGHTS_PATH, monitor='val_f1_m', save_best_only=True, mode='max')
# 训练
model.fit_generator(train_generator.forfit(), steps_per_epoch=train_generator.steps, epochs=settings.EPOCHS,
validation_data=dev_generator.forfit(), validation_steps=dev_generator.steps,
callbacks=[checkpoint, ])
# 测试
print('测试集结果：')
loss, accuracy, f1_score, precision, recall = model.evaluate_generator(test_generator.forfit(),
steps=test_generator.steps)
print('loss =', loss)
print('accuracy =', accuracy)
print('f1 score =', f1_score)
print('precision =', precision)
print('recall =', recall)
# plt.imshow(bx[-1].astype(np.uint8))
test_bson_file = open(r.test_bson_path, "rb")
test_datagen = ImageDataGenerator()
test_gen = BSONIterator(test_bson_file,
r.test_images_df,
r.test_offsets_df,
num_classes,
test_datagen,
batch_size=batch_size,
with_labels=False,
shuffle=False,
keep_indices=True)
# To train the model:
print("fitting\n")
steps_per_epoch = num_train_images // batch_size
model.fit_generator(train_gen,
steps_per_epoch=steps_per_epoch,
epochs=1,
validation_data=val_gen,
validation_steps=num_val_images,
workers=8)
num_test_samples = len(r.test_images_df)
print("predicting")
predictions = model.predict_generator(test_gen,
steps=1 + num_test_samples // batch_size,
workers=8)
epochs = 30
batch_size = 64
model = model(tiles_per_dim, image_shape, sinkhorn_on, weight_decay, dropout)
sgd = keras.optimizers.SGD(lr=initial_lr, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy',
optimizer=keras.optimizers.Adam(lr=initial_lr),
metrics=['accuracy'])
print(model.summary())
datagen = data_generator(X_train, y_train, batch_size=batch_size)
vdatagen = data_generator(X_test, y_test, batch_size=batch_size)
history = model.fit_generator(generator=datagen,
steps_per_epoch=X_train.shape[0] // batch_size,
validation_data=vdatagen,
validation_steps=X_test.shape[0] // batch_size,
epochs=epochs,
callbacks=[LearningRateScheduler(schedule)])
# summarize history for accuracy
plt.plot(history.history['acc'])
plt.plot(history.history['val_acc'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
# summarize history for loss
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
epochs = 30
batch_size = 64
model = model(tiles_per_dim, image_shape, sinkhorn_on, weight_decay, dropout)
sgd = keras.optimizers.SGD(lr=initial_lr, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy',
optimizer=keras.optimizers.Adam(lr=initial_lr),
metrics=['accuracy'])
print(model.summary())
datagen = data_generator(X_train, y_train, batch_size=batch_size)
vdatagen = data_generator(X_test, y_test, batch_size=batch_size)
history = model.fit_generator(generator=datagen,
steps_per_epoch=X_train.shape[0] // batch_size,
validation_data=vdatagen,
validation_steps=X_test.shape[0] // batch_size,
epochs=epochs,
callbacks=[LearningRateScheduler(schedule)])
# summarize history for accuracy
plt.plot(history.history['acc'])
plt.plot(history.history['val_acc'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
# summarize history for loss
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')