def train_model(struct_list, opt, no_epochs, test_size, data, images ):
split = train_test_split(data, images, test_size=test_size, random_state=42)
(trainAttrX, testAttrX, trainImagesX, testImagesX) = split
out_scaler = MinMaxScaler()
trainY = out_scaler.fit_transform(trainAttrX["price"].values.reshape(-1, 1))
testY = out_scaler.fit_transform(testAttrX["price"].values.reshape(-1, 1))
trainAttrX, testAttrX, labelizer, cs = process_house_attributes(data, trainAttrX, testAttrX)
mlp = create_mlp(trainAttrX.shape[1], regress=False)
cnn = create_cnn(64, 64, 3, struct_list, regress=False)
combinedInput = concatenate([mlp.output, cnn.output])
x = Dense(4, activation="relu", name="fc_0")(combinedInput)
x = Dense(1, activation="linear", name="fc_1")(x)
model = Model(inputs=[mlp.input, cnn.input], outputs=x)
model.compile(loss="mean_absolute_percentage_error", optimizer=opt, metrics=['accuracy'])
history_callback = model.fit( [trainAttrX, trainImagesX], trainY,
validation_data=([testAttrX, testImagesX], testY),
epochs=no_epochs, batch_size=25)
hist_df = pd.DataFrame(history_callback.history)
return (model, hist_df, labelizer, cs, out_scaler)
# find the largest house price in the training set and use it to # scale our house prices to the range [0, 1] (will lead to better # training and convergence) maxPrice = trainAttrX["price"].max() trainY = trainAttrX["price"] / maxPrice testY = testAttrX["price"] / maxPrice # process the house attributes data by performing min-max scaling # on continuous features, one-hot encoding on categorical features, # and then finally concatenating them together (trainAttrX, testAttrX) = datasets.process_house_attributes(df, trainAttrX, testAttrX) # create the MLP and CNN models mlp = models.create_mlp(trainAttrX.shape[1], regress=False) cnn = models.create_cnn(64, 64, 3, regress=False) # create the input to our final set of layers as the *output* of both # the MLP and CNN combinedInput = concatenate([mlp.output, cnn.output]) # our final FC layer head will have two dense layers, the final one # being our regression head x = Dense(4, activation="relu")(combinedInput) x = Dense(1, activation="linear")(x) # our final model will accept categorical/numerical data on the MLP # input and images on the CNN input, outputting a single value (the # predicted price of the house) model = Model(inputs=[mlp.input, cnn.input], outputs=x)
# the data for training and the remaining 25% for testing
split = train_test_split(df, images, test_size=0.25, random_state=42)
(trainAttrX, testAttrX, trainImagesX, testImagesX) = split
# find the largest house price in the training set and use it to
# scale our house prices to the range [0, 1] (will lead to better
# training and convergence)
maxPrice = trainAttrX["price"].max()
trainY = trainAttrX["price"] / maxPrice
testY = testAttrX["price"] / maxPrice
# create our Convolutional Neural Network and then compile the model
# using mean absolute percentage error as our loss, implying that we
# seek to minimize the absolute percentage difference between our
# price *predictions* and the *actual prices*
model = models.create_cnn(64, 64, 3, regress=True)
opt = Adam(lr=1e-3, decay=1e-3 / 200)
model.compile(loss="mean_absolute_percentage_error", optimizer=opt)
# train the model
print("[INFO] training model...")
model.fit(trainImagesX,
trainY,
validation_data=(testImagesX, testY),
epochs=200,
batch_size=8)
# make predictions on the testing data
print("[INFO] predicting house prices...")
preds = model.predict(testImagesX)
trainIDs = train.index.values
testIDs = test.index.values
trainX = []
for i in range(len(trainY)):
id = trainIDs[i]
trainX.append(images[i])
trainX = np.array(trainX)
testX = []
for i in range(len(testY)):
id = testIDs[i]
testX.append(images[i])
testX = np.array(testX)
model = models.create_cnn(trainX[0].shape, regress=True)
opt = Adam(
lr=1e-3, decay=1e-3 / 200
) # lr --> Learnrate decay ---> absenken der Lernrate nach jeder Epoche
model.compile(loss="mean_absolute_percentage_error", optimizer=opt)
history = model.fit(trainX,
trainY,
validation_data=(testX, testY),
epochs=1000,
batch_size=20)
score = model.evaluate(trainX, trainY, verbose=0)
print('Test loss:', score)
'''
# Plot training & validation accuracy values
inputPath = os.path.sep.join([args["dataset"], "cat_data.txt"])
df = f1.load_cat_attributes(inputPath)
print(df)
print("[INFO] loading cat images...")
images = f1.load_cat_images(df, args["dataset"])
images = images / 255.0
split = train_test_split(df, images, test_size=0.10, random_state=42)
(trainAttrX, testAttrX, trainImagesX, testImagesX) = split
maxPrice = trainAttrX["engagement"].max()
trainY = trainAttrX["engagement"] / maxPrice
testY = testAttrX["engagement"] / maxPrice
model = f2.create_cnn(256, 256, 3, regress=True)
opt = Adam(lr=1e-3, decay=1e-3 / 200)
model.compile(loss="mean_absolute_percentage_error", optimizer=opt)
print("[INFO] training model...")
model.fit(trainImagesX,
trainY,
validation_data=(testImagesX, testY),
epochs=50,
batch_size=8)
print("[INFO] predicting engagement...")
preds = model.predict(testImagesX)
diff = preds.flatten() - testY
percentDiff = (diff / testY) * 100
def main():
print('Training the join cardinality estimator')
is_train = True
num_rows, num_columns = 16, 16
# target = 'join_selectivity'
target = 'mbr_tests_selectivity'
datasets_features_path = 'data/spatial_descriptors/spatial_descriptors_small_datasets.csv'
datasets_histograms_path = 'data/histograms/small_datasets'
join_results_path = 'data/join_results/join_results_small_datasets_no_bit.csv'
features_df = datasets.load_datasets_feature(datasets_features_path)
join_data, ds1_histograms, ds2_histograms, ds_all_histogram, ds_bops_histogram = datasets.load_join_data(
features_df, join_results_path, datasets_histograms_path, num_rows,
num_columns)
train_attributes, test_attributes, ds1_histograms_train, ds1_histograms_test, ds2_histograms_train, ds2_histograms_test, ds_all_histogram_train, ds_all_histogram_test, ds_bops_histogram_train, ds_bops_histogram_test = train_test_split(
join_data,
ds1_histograms,
ds2_histograms,
ds_all_histogram,
ds_bops_histogram,
test_size=0.20,
random_state=42)
# train_attributes, val_attributes, ds1_histograms_train, ds1_histograms_val, ds2_histograms_train, ds2_histograms_val, ds_all_histogram_train, ds_all_histogram_val = train_test_split(
# train_attributes, ds1_histograms_train, ds2_histograms_train, ds_all_histogram_train, test_size=0.20, random_state=32)
num_features = len(train_attributes.columns) - 10
# print (join_data)
X_train = pd.DataFrame.to_numpy(
train_attributes[[i for i in range(num_features)]])
X_test = pd.DataFrame.to_numpy(
test_attributes[[i for i in range(num_features)]])
y_train = train_attributes[target]
y_test = test_attributes[target]
# y_train = train_attributes['result_size']
# y_test = test_attributes['result_size']
mlp = models.create_mlp(X_train.shape[1], regress=False)
cnn1 = models.create_cnn(num_rows, num_columns, 1, regress=False)
# cnn2 = models.create_cnn(num_rows, num_columns, 1, regress=False)
# cnn3 = models.create_cnn(num_rows, num_columns, 1, regress=False)
# combined_input = concatenate([mlp.output, cnn1.output, cnn2.output, cnn3.output])
combined_input = concatenate([mlp.output, cnn1.output])
x = Dense(4, activation="relu")(combined_input)
x = Dense(1, activation="linear")(x)
# model = Model(inputs=[mlp.input, cnn1.input, cnn2.input, cnn3.input], outputs=x)
model = Model(inputs=[mlp.input, cnn1.input], outputs=x)
EPOCHS = 40
LR = 1e-2
# opt = Adam(lr=1e-4, decay=1e-4 / 200)
opt = Adam(lr=LR, decay=LR / EPOCHS)
model.compile(loss="mean_absolute_percentage_error", optimizer=opt)
# print (model.summary())
# train the model
if is_train:
print("[INFO] training model...")
# model.fit(
# [X_train, ds1_histograms_train, ds2_histograms_train], y_train,
# validation_data=([X_test, ds1_histograms_test, ds2_histograms_test], y_test),
# epochs=EPOCHS, batch_size=128)
model.fit([X_train, ds_bops_histogram_train],
y_train,
validation_data=([X_test, ds_bops_histogram_test], y_test),
epochs=EPOCHS,
batch_size=256)
model.save('trained_models/model.h5')
model.save_weights('trained_models/model_weights.h5')
else:
model = keras.models.load_model('trained_models/model.h5')
model.load_weights('trained_models/model_weights.h5')
print('Test on small datasets')
y_pred = model.predict([X_test, ds_bops_histogram_test])
print('r2 score: {}'.format(r2_score(y_test, y_pred)))
diff = y_pred.flatten() - y_test
percent_diff = (diff / y_test)
abs_percent_diff = np.abs(percent_diff)
# test_attributes['join_selectivity_pred'] = y_pred
# test_attributes['percent_diff'] = abs_percent_diff
# test_attributes.to_csv('prediction_small.csv')
# compute the mean and standard deviation of the absolute percentage
# difference
mean = np.mean(abs_percent_diff)
std = np.std(abs_percent_diff)
print('mean = {}, std = {}'.format(mean, std))
print('Test on large datasets')
datasets_features_path = 'data/spatial_descriptors/spatial_descriptors_large_datasets.csv'
datasets_histograms_path = 'data/histograms/large_datasets'
join_results_path = 'data/join_results/join_results_large_datasets_no_bit.csv'
features_df = datasets.load_datasets_feature(datasets_features_path)
join_data, ds1_histograms, ds2_histograms, ds_all_histogram, ds_bops_histogram = datasets.load_join_data(
features_df, join_results_path, datasets_histograms_path, num_rows,
num_columns)
X_test = pd.DataFrame.to_numpy(join_data[[i for i in range(num_features)]])
y_test = join_data[target]
y_pred = model.predict([X_test, ds_bops_histogram])
print('r2 score: {}'.format(r2_score(y_test, y_pred)))
diff = y_pred.flatten() - y_test
percent_diff = (diff / y_test)
abs_percent_diff = np.abs(percent_diff)
mean = np.mean(abs_percent_diff)
std = np.std(abs_percent_diff)
print('mean = {}, std = {}'.format(mean, std))
trainY = [
1 / (max_steer - min_steer) * (trainValX[:, 0] - min_steer),
1 / (max_throttle - min_throttle) * (trainValX[:, 1] - min_throttle)
] # obtained from y = ax + b ...
testY = [
1 / (max_steer - min_steer) * (testValX[:, 0] - min_steer),
1 / (max_throttle - min_throttle) * (testValX[:, 1] - min_throttle)
] # obtained from y = ax + b ...
# transformation
trainY = np.array(trainY).T
testY = np.array(testY).T
# the fun begins here!!!
# create model
model = models.create_cnn(IMG_WIDTH, IMG_HEIGHT)
# compile model
model.compile(loss='mean_squared_error',
optimizer='adam',
metrics=['accuracy'])
# model summary
print('Model summary:')
model.summary()
# train the model
model.fit(trainImgX,
trainY,
validation_data=(testImgX, testY),
epochs=4,
# Find the largest house price in the training set and use it to scale our house prices # to the range [0, 1] (which will lead to better training and convergence) maxPrice = trainAttrX["price"].max() trainY = trainAttrX["price"] / maxPrice testY = testAttrX["price"] / maxPrice # Process the house attributes data by performing min-max scaling # on continunous features, one-shot encoding on categorical features, # and then finally concatenating them together (trainAttrX, testAttrX) = dataset.process_house_attributes(df, trainAttrX, testAttrX) # Create the MLP and CNN models mlp = models.create_mlp(dim=trainAttrX.shape[1], regress=False) cnn = models.create_cnn(width=64, height=64, channel=3, regress=False) # Create the input to our final set of layers as the "output" of both the MLP and CNN combinedInput = concatenate([mlp.output, cnn.output]) # Final FC layer head will have two dense layers, the final one is our regression head x = Dense(units=4, activation="relu")(combinedInput) x = Dense(units=1, activation="linear")(x) # Final model model = Model(inputs=[mlp.input, cnn.input], outputs=x) # Complie the model using MAPE as loss function optimizer = Adam(lr=1e-3, decay=1e-3 / 200) model.compile(loss="mean_absolute_percentage_error", optimizer=optimizer)
chanDim = -1 class_list = ["aluminium", "cardboard", "glass", "paper", "plastic", "thrash"] FC_LAYERS = [1024, 1024] dropout = 0.5 EPOCHS = 8 #200 BS = 8 INIT_LR = 1e-3 #INIT_LR/EPOCHS = (1e-3)/8 HEIGHT = 224 WIDTH = 224 ######################################################################################################### CHOOSE TRAINING OR RETRAINING WITH AND MODELS.PY #(A) CREATE the MLP and CNN models #CHOOSE EITHER A OR B mlp = models.create_mlp(trainAttrX.shape[1], regress=False) cnn = models.create_cnn(dropout, FC_LAYERS, len(class_list), regress=False) # create the input to our final set of layers as the *output* of both # the MLP and CNN combinedInput = concatenate([mlp.output, cnn.output]) # our final FC layer head will have two dense layers, the final one # being our regression head x = Dense(8, activation="relu")(combinedInput) # New softmax layer x = Dense(len(class_list), activation='softmax')(x) # our final model will accept numerical data on the MLP # input and images on the CNN input, outputting a predicted category of waste) model = Model(inputs=[mlp.input, cnn.input], outputs=x)
def train_target_score_model(imagesPath, modelPath, tensorboard_gcs_logs,
epochs_count, batch_size):
train_images_dir = imagesPath
scores_file = os.path.join(imagesPath, "scores.json")
IMAGE_SIZE = 128
with open(scores_file) as f:
train_data = json.load(f)
labels = []
img_data = []
img_index = 0
#tensorboard_gcs_logs = 'gs://dpa23/tarsanlogs'
img_to_show = []
def add_image(img, rotation, score):
labels.append(score)
if rotation != 0:
r_image = img.rotate(rotation)
else:
r_image = img
img_array = img_to_array(r_image) / 255.0
img_data.append(img_array)
for file_name, score in train_data.items():
full_file_name = os.path.join(train_images_dir, file_name)
print("Loading {} with score {}".format(full_file_name, score))
img = load_img(full_file_name,
color_mode="grayscale",
target_size=(IMAGE_SIZE, IMAGE_SIZE),
interpolation='bilinear')
if img_index < 10:
buffered = BytesIO()
img.save(buffered, format="PNG")
img_base64 = base64.b64encode(buffered.getvalue()).decode("utf-8")
img_to_show.append((img_base64, score, 0.0))
add_image(img, 0, score)
add_image(img, 45, score)
add_image(img, 90, score)
add_image(img, 135, score)
img_index = img_index + 1
np_labels = np.asfarray(labels)
np_image_data = np.asfarray(img_data)
# partition the data into training and testing splits using 75% of
# the data for training and the remaining 25% for testing
split = train_test_split(np_labels,
np_image_data,
test_size=0.25,
random_state=42)
(trainAttrX, testAttrX, trainImagesX, testImagesX) = split
# find the largest score in the training set and use it to
# scale the scores to the range [0, 1]
maxScore = trainAttrX.max()
trainY = trainAttrX / maxScore
testY = testAttrX / maxScore
# create our Convolutional Neural Network and then compile the model
# using mean absolute percentage error as our loss, implying that we
# seek to minimize the absolute percentage difference between our
# score *predictions* and the *actual score*
model = models.create_cnn(IMAGE_SIZE, IMAGE_SIZE, 1, regress=True)
opt = Adam(lr=1e-3, decay=1e-3 / 200)
model.compile(loss="mean_absolute_percentage_error", optimizer=opt)
# Define Tensorboard as a Keras callback
tensorboard = TensorBoard(log_dir=tensorboard_gcs_logs + '/' +
datetime.now().strftime("%Y%m%d-%H%M%S"),
histogram_freq=5,
write_images=True,
update_freq='epoch')
keras_callbacks = []
if len(tensorboard_gcs_logs) > 2:
keras_callbacks = [tensorboard]
# train the model
print("[INFO] training model...")
model.fit(trainImagesX,
trainY,
validation_data=(testImagesX, testY),
epochs=epochs_count,
batch_size=batch_size,
callbacks=keras_callbacks)
print("[INFO] saving model to {} ...".format(modelPath))
model.save(modelPath)
# make predictions on the testing data
print("[INFO] predicting scores prices...")
preds = model.predict(testImagesX)
# compute the difference between the *predicted* scores and the
# *actual* scores, then compute the percentage difference and
# the absolute percentage difference
diff = preds.flatten() - testY
percentDiff = (diff / testY) * 100
absPercentDiff = np.abs(percentDiff)
# compute the mean and standard deviation of the absolute percentage
# difference
mean = np.mean(absPercentDiff)
std = np.std(absPercentDiff)
# finally, show some statistics on our model
print("[INFO] avg. score: {}, std score: {}".format(
np_labels.mean(), np_labels.std()))
metrics = {
'metrics': [{
'name': 'diff-mean',
'numberValue': mean,
'format': "PERCENTAGE",
}]
}
with open('/mlpipeline-metrics.json', 'w') as f:
json.dump(metrics, f)
img_html = '<table><tr><th>Target</th><th>Actual Score</th><th>Predicted Score</th></tr>'
for (img_b64, s1, s2) in img_to_show:
html_line = '<tr><td><img src="data:image/png;base64, {}" alt="Target Example"></td><td>{}</td><td>{}</td></tr>'.format(
img_b64, s1, s2)
img_html = img_html + html_line
img_html = img_html + '</table>'
metadata = {
'outputs': [{
'storage': 'inline',
'source': 'Markdown text',
'type': 'markdown',
}, {
'type': 'web-app',
'storage': 'inline',
'source': img_html,
}, {
'type': 'tensorboard',
'source': tensorboard_gcs_logs,
}]
}
with open('/mlpipeline-ui-metadata.json', 'w') as f:
json.dump(metadata, f)
print("[INFO] mean: {:.2f}%, std: {:.2f}%".format(mean, std))