def training_evaluation(average_model, model_params_list, st_model):
"""Validate the training error"""
average_prediction_mae = mean([
mean_absolute_error(param.y, param.plane.predict(param.X))
for param in model_params_list
])
average_prediction_pc = mean([
get_pc(param.y, param.plane.predict(param.X))
for param in model_params_list
])
average_model_mae = mean([
mean_absolute_error(param.y, average_model.plane.predict(param.X))
for param in model_params_list
])
average_model_pc = mean([
get_pc(param.y, average_model.plane.predict(param.X))
for param in model_params_list
])
st_model_mae = mean([
mean_absolute_error(param.y,
st_model.predict(param.SA, param.TA, param.X))
for param in model_params_list
])
st_model_pc = mean([
get_pc(param.y, st_model.predict(param.SA, param.TA, param.X))
for param in model_params_list
])
print(f"Average individual MAE {average_prediction_mae}")
print(f"Average individual PC {average_prediction_pc}")
print(f"Average model MAE {average_model_mae}")
print(f"Average model PC {average_model_pc}")
print(f"ST model MAE {st_model_mae}")
print(f"ST model PC {st_model_pc}")
def validate_correction_factor(name, df, trainer_handle, model_parameter,
model_handle):
"""Validate a correction factor with a given bitrate model and trainer handle"""
trainer = trainer_handle(df, model_handle, model_parameter)
params = trainer_handle.params_from_mp(model_parameter)
expected_cf, predicted_cf = trainer.validate(params)
print_rmse_and_pc(
f"{name} validation results",
get_rmse(expected_cf, predicted_cf),
get_pc(expected_cf, predicted_cf),
)
def evaluate_block(df, prefix):
"""Evaluate for one video block"""
base_key = f"{prefix}{BASE_KEY}"
filter_key = f"{prefix}{FILTER_KEY}"
# Plot bitrate
plt.clf()
cycol, filter_keys = plot_bitrate(base_key, df, filter_key)
plt.savefig(f"{prefix}bitrate")
# Plot QPs with matching bitrate
plt.clf()
base, lines, eval_data = get_qp_match(base_key, cycol, df, filter_key,
filter_keys)
plt.savefig(f"{prefix}QPs")
X, f, linear_regression, y = fit_plane(base, lines, prefix)
plt.savefig(f"{prefix}plane")
# evaluate error
linear_regression_mae = mean_absolute_error(y,
linear_regression.predict(X))
linear_regression_pc = get_pc(y, linear_regression.predict(X))
plane_error = mean_absolute_error(
y,
predict_with_plane_equation(
linear_regression.intercept_,
linear_regression.coef_[0],
linear_regression.coef_[1],
X,
),
)
tqdm.write(f"\nPrefix: {prefix}")
tqdm.write(f"Linear regression MAE: {linear_regression_mae}")
tqdm.write(f"Linear regression PC: {linear_regression_pc}")
tqdm.write(
f"FS = {linear_regression.intercept_} + {linear_regression.coef_[0]} * QP_i + "
f"{linear_regression.coef_[1]} * QP_SF")
tqdm.write(f"Plane equation MAE: {plane_error}")
return (
ModelParams(
prefix,
linear_regression.intercept_,
linear_regression.coef_,
eval_data[SA_KEY].iloc[0],
eval_data[TA_KEY].iloc[0],
linear_regression,
X,
y,
),
eval_data,
)
def train_correction_factor(name, df, trainer_handle, model_handle,
init_mode_parameter):
"""Train a correction factor with a given bitrate model and trainer handle"""
trainer = trainer_handle(df, model_handle, init_mode_parameter)
param_min = trainer.train()
expected_cf, predicted_cf = trainer.validate(param_min)
print_rmse_and_pc(
f"{name} training results",
get_rmse(expected_cf, predicted_cf),
get_pc(expected_cf, predicted_cf),
)
return param_min
def plot_bitrates(bitrate_measured, bitrate_estimated, key, decimals=-1):
"""
Plot measured over estimated bitrates
By default rounded to -1 decimals as plot as larger plot cannot be handled by latex
Accuracy is still fine as plot is shown in very small size
"""
rmse = get_rmse(bitrate_measured, bitrate_estimated)
pc = get_pc(bitrate_measured, bitrate_estimated)
plt.figure()
estimated = np.around(bitrate_estimated, decimals=decimals)
measured = np.around(bitrate_measured, decimals=decimals)
plt.scatter(estimated, measured, marker="x")
plt.xlabel("Estimated Bitrate [kBit/s]")
plt.ylabel("Measured Bitrate [kBit/s]")
plt.title(
f"Bitrate model comparison {key} (RMSE: {rmse:.2f}, PC: {pc:.5f})")
def validate(model, dfw, input_scaler, output_scaler, data_name="test"):
"""Validate the trained ML model on the test set"""
in_test = input_scaler.transform(dfw.get_ml_input())
in_test = tf.convert_to_tensor(in_test, dtype=tf.float32)
out_test = dfw.get_ml_output()
print(f"Evaluating on {data_name} data...")
results = model.evaluate(in_test, output_scaler.transform(out_test))
print(f"{data_name} loss, accuracy: ", results)
print(f"Predicting on {data_name} data...")
output = model.predict(in_test)
out_prediction = output_scaler.inverse_transform(output)
max_bitrates = dfw.get_ml_max_output()
print_rmse_normalized_rmse_and_pc(
"Bitrate average",
get_rmse(out_test, out_prediction) * 0.001,
get_rmse(
out_test.transpose() / max_bitrates,
out_prediction.transpose() / max_bitrates,
),
get_pc(out_test.transpose(), out_prediction.transpose()),
)
def validate(dfw, model_parameter, model_handle, full_evaluation):
"""
Validate all supported correction factors
"""
video_keys = dfw.video_keys
bitrate_model = model_handle(model_parameter)
rmses = []
nrmses = []
pcs = []
for key in video_keys.tolist():
bitrate_measured, bitrate_estimated = validate_bitrates_for_video(
get_df_for_evaluation(dfw, model_handle, full_evaluation),
bitrate_model,
key,
)
rmses.append(get_rmse(bitrate_measured, bitrate_estimated))
nrmses.append(rmses[-1] / max(bitrate_measured))
pcs.append(get_pc(bitrate_measured.T, bitrate_estimated.T))
print_rmse_normalized_rmse_and_pc(f"Bitrate validation {key}",
rmses[-1], nrmses[-1], pcs[-1])
validate_correction_factor("Rmax", dfw.include(), RMaxTrainer,
model_parameter, model_handle)
validate_correction_factor(
"SCF",
dfw.include(variable_qp=True),
SCFTrainer,
model_parameter,
model_handle,
)
validate_correction_factor(
"TCF",
dfw.include(variable_rate=True),
TCFTrainer,
model_parameter,
model_handle,
)
validate_correction_factor(
"NCF",
dfw.include(variable_gop=True),
NCFTrainer,
model_parameter,
model_handle,
)
validate_correction_factor(
"RCF",
dfw.include(variable_res=True),
RCFTrainer,
model_parameter,
model_handle,
)
validate_correction_factor(
"GCF",
dfw.include(variable_k_size=True),
GCFTrainer,
model_parameter,
model_handle,
)
df_gauss = dfw.include(variable_k_size=True, variable_sigma=True)
validate_correction_factor(
"SDCF",
df_gauss.loc[df_gauss[KEYS.KSIZE] == 3],
SDCFTrainer,
model_parameter,
model_handle,
)
print_rmse_normalized_rmse_and_pc(
"Bitrate validation average",
statistics.mean(rmses),
statistics.mean(nrmses),
statistics.mean(pcs),
)