def train_separate_batch(i):
print(
"Ok. Train Batch {}, First load previous model, then continue training."
.format(i))
current_data = total_batches[i]
split_point = int(0.8 * PARA_LEN)
x = []
y = []
for j in tqdm(range(len(current_data))):
features = cia.convert_img(current_data[j][2])
label = to_categorical(int(current_data[j][3]), num_classes=7)
x.append(features)
y.append(label)
x_train = x[:split_point]
y_train = y[:split_point]
x_val = x[split_point:]
y_val = y[split_point:]
if (i == 0):
model = training_model("VGG16")
else:
model = load_model("separate_batch_{}_of_{}.h5".format(i, SPLIT_TIMES))
history = model.fit(x_train,
y_train,
epochs=5,
batch_size=64,
validation_data=(x_val, y_val))
model.save("separate_batch_{}_of_{}.h5".format(i, SPLIT_TIMES))
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)
plt.plot(epochs, loss, 'bo', label='Training loss')
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()
plt.show()
plt.savefig("Loss_batch_{}_of_{}.png".format(i, SPLIT_TIMES))
plt.clf()
acc = history.history['acc']
val_acc = history.history['val_acc']
plt.plot(epochs, acc, 'bo', label='Training acc')
plt.plot(epochs, val_acc, 'b', label='Validation acc')
plt.title('Training and Validation accuracy')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.legend()
plt.show()
plt.savefig("Accuracy_batch_{}_of_{}.png".format(i, SPLIT_TIMES))
def data_generation(self, batch_datas):
images = []
labels = []
# 生成数据
for i, data in enumerate(batch_datas):
#x_train数据
feautres = cia.convert_img(data[2])
images.append(feautres)
#y_train数据
labels.append(to_categorical(int(data[self.label]),num_classes=self.num_classes))
# print("data_generation: ",np.array(images).shape,np.array(labels).shape)
return np.array(images), np.array(labels)
def train_full_batch():
print("Ok. Train Full Batch, start training.")
current_data = rows
split_point = int(0.8 * PARA_LEN)
x = []
y = []
for j in tqdm(range(len(current_data))):
features = cia.convert_img(current_data[j][2])
label = to_categorical(int(current_data[j][3]), num_classes=7)[0]
x.append(features)
y.append(label)
x = np.array(x)
y = np.array(y)
print("Array conversion finished.")
x_train = x[:split_point]
y_train = y[:split_point]
x_val = x[split_point:]
y_val = y[split_point:]
train_x_word = word2id(max_len=20)
finished = 0
if finished:
train_x_word = pickle.load(open('r-benefits-text-features', 'rb'))
else:
print("Text Features not extracted, start extracting...")
train_x_word = word2id(max_len=20)
pickle.dump(train_x_word, open('r-benefits-text-features', 'wb'))
print("start padding sequences")
x_nlp = sequence.pad_sequences(train_x_word,
maxlen=20,
dtype='int32',
padding='post',
truncating='post',
value=UNK)
print("padding finished.")
x_nlp_train = x_nlp[:split_point]
x_nlp_val = x_nlp[split_point:]
model = combined_model([256, 128, 32], 7)
history = model.fit([x_train, x_nlp_train],
y_train,
epochs=5,
batch_size=64,
validation_data=([x_val, x_nlp_val], y_val))
model.save("full_batch.h5")
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)
plt.plot(epochs, loss, 'bo', label='Training loss')
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()
plt.show()
plt.savefig("Training_and_validation_loss_full_batch{}.png")
plt.clf()
acc = history.history['acc']
val_acc = history.history['val_acc']
plt.plot(epochs, acc, 'bo', label='Training acc')
plt.plot(epochs, val_acc, 'b', label='Validation acc')
plt.title('Training and Validation accuracy')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.legend()
plt.show()
plt.savefig("Training_and_validation_accuracy_full_batch{}.png")
image_class = int(rows[i][3])
image_url = rows[i][2]
if image_class == 2:
tmp_result = cgia.convert_and_generage_img(image_url, 10)
for item in tmp_result:
data_x.append(item)
for j in range(11):
data_y.append(to_categorical(2, num_classes=7))
elif image_class == 5:
tmp_result = cgia.convert_and_generage_img(image_url, 5)
for item in tmp_result:
data_x.append(item)
for j in range(6):
data_y.append(to_categorical(5, num_classes=7))
else:
data_x.append(cia.convert_img(image_url))
data_y.append(to_categorical(image_class, num_classes=7))
print("-" * 30, "Start converting data_x to array", "-" * 30)
data_x = np.array(data_x)
print("-" * 30, "Start converting data_y to array.", "-" * 30)
data_y = np.array(data_y)
print("-" * 30, "Conversion to array finished.", "-" * 30)
print("-" * 30, "Start train-test splitting.", "-" * 30)
X_train_val, X_test, y_train_val, y_test = train_test_split(data_x,
data_y,
test_size=0.2,
random_state=42)
print("-" * 30, "Splitting finished.", "-" * 30)
for i in range(times):
batches.append(whole_data[start:end])
start += num_per_batch
end += num_per_batch
return batches
total_batches = split_batches(rows,SPLIT_TIMES)
def train_separate_batch(i)
print("Ok. Train Batch {}, First load previous model, then continue training.".format(i))
current_data = total_batches[i]
split_point = int(0.8 * PARA_LEN)
x = []
y = []
for j in range(len(current_data)):
features = cia.convert_img(current_data[j][2])
label = to_categorical(int(current_data[j][3]),num_classes=7)
x.append(features)
y.append(label)
x_train = x[:split_point]
y_train = y[:split_point]
x_val = x[split_point:]
y_val = y[split_point:]
if (i == 0):
model = training_model("VGG16")
else:
model = load_model("separate_batch{}.h5".format(i))
history = model.fit(x_train, y_train, epochs=5, batch_size=64,validation_data=(x_val,y_val))
model.save("separate_batch{}.h5".format(i))
