def get_metrics(model_path, X_val_path, y_val_path):
model = tf.keras.models.load_model(
model_path, custom_objects={'r2': selectMetric('r2')})
X_test = np.load(X_val_path)
y_test = np.load(y_val_path)
y_pred = model.predict(X_test)
#Averaging down the columns first (metrics), which converts it to 1 error per metric, then averages that (one row)
mse = np.mean(np.mean(np.square(y_pred - y_test), axis=0))
rmse = np.sqrt(mse)
normX_testV = np.mean(np.mean(np.square(y_test), axis=0))
#relative to the full scale of all the test data - 3x bigger because it's one number scaling across all metrics
mse_relative = np.mean(np.mean(np.square(y_pred - y_test),
axis=0)) / normX_testV
mspe = np.mean(np.mean(np.square(y_pred - y_test),
axis=0)) / normX_testV ## ask bethany
rmse_relative = np.sqrt(mspe)
R = sp.stats.pearsonr(y_test.flatten(), y_pred.flatten())[0]
MAE = np.mean(np.abs(y_pred.flatten() - y_test.flatten()))
RMSE = np.sqrt(np.power(y_test.flatten() - y_pred.flatten(), 2).mean())
""""""
print("R:", R)
print("MAE:", MAE)
print("RMSE:", RMSE)
return R, MAE, RMSE
def get_metrics(model_path, X_val_path, y_val_path, predict_column):
if "RF_" in model_path:
model = pickle.load(open(model_path, 'rb'))
else:
model = tf.keras.models.load_model(
model_path, custom_objects={'r2': selectMetric('r2')})
X_test = np.load(X_val_path)
y_test = np.load(y_val_path)
y_pred = model.predict(X_test)
if predict_column:
y_pred = y_pred[:, predict_column]
y_test = y_test[:, predict_column]
#import pdb; pdb.set_trace()
#Averaging down the columns first (metrics), which converts it to 1 error per metric, then averages that (one row)
mse = np.mean(np.mean(np.square(y_pred - y_test), axis=0))
rmse = np.sqrt(mse)
#load scaler
from sklearn.externals.joblib import load
ipc_scaler = load('scalers/std_scaler_ipc.bin')
y_pred = ipc_scaler.inverse_transform(y_pred)
y_test = ipc_scaler.inverse_transform(y_test)
def mean_absolute_percentage_error(y_true, y_pred):
"""y_true, y_pred = np.array(y_true), np.array(y_pred)
return np.mean(np.abs((y_true - y_pred) / y_true)) * 100"""
diffs = []
for v in range(len(y_true)):
#diffs.append(abs(max(y_pred[v], 0.0) - y_true[v]) / y_true[v])
diffs.append(abs(y_pred[v] - y_true[v]) / y_true[v])
return np.mean(diffs) * 100.0, np.std(diffs) * 100.0
normX_testV = np.mean(np.mean(np.square(y_test), axis=0))
#relative to the full scale of all the test data - 3x bigger because it's one number scaling across all metrics
mse_relative = np.mean(np.mean(np.square(y_pred - y_test),
axis=0)) / normX_testV
mspe = np.mean(np.mean(np.square(y_pred - y_test),
axis=0)) / normX_testV ## ask bethany
rmse_relative = np.sqrt(mspe)
R = sp.stats.pearsonr(y_test.flatten(), y_pred.flatten())[0]
MAE = np.mean(np.abs(y_pred.flatten() - y_test.flatten()))
RMSE = np.sqrt(np.power(y_test.flatten() - y_pred.flatten(), 2).mean())
R2 = r2_score(y_test, y_pred)
MAPE, MAPE_std = mean_absolute_percentage_error(y_test, y_pred)
""""""
print("R:", R)
print("MAE:", MAE)
print("RMSE:", RMSE)
print("R2:", R2)
print("MAPE:", MAPE)
print("MAPE-std:", MAPE_std)
return R, MAE, RMSE, R2, MAPE, MAPE_std
import scipy as sp
#selection type can be either Random or AL or C
#change unique to best or best to unique
selection_type = 'C'
if selection_type == 'AL':
type_c = 'best'
else:
type_c = 'unique'
val_summary_dict = {'Selection':[],'Percent':[],'Application':[],'MAPE':[]}
model_path = '/Users/yzamora/power/nvidia_gpus/all_apps/deephyper_final_models/' + selection_type +'_one_20-POST_' + type_c + '.h5'
#model_path = '/Users/yzamora/power/nvidia_gpus/all_apps/deephyper_models/best_model_18_per20.h5'
model = tf.keras.models.load_model(model_path,
custom_objects={
'r2': selectMetric('r2')
}
)
# When using .sav models
###model = pickle.load(open(model_path,'rb'))
"""
X_test = np.load('/Users/yzamora/active_learning/X_val_RANDOM_20per.npy')
y_test = np.load('/Users/yzamora/active_learning/y_val_RANDOM_20per.npy')
"""
app_single_name = "leukocyte"
##X_test = np.load('/Users/yzamora/power/nvidia_gpus/all_apps/specified_application_indices/Random_val_indices/Random_20per_' + app_single_name + '_x_val.npy')
##y_test = np.load('/Users/yzamora/power/nvidia_gpus/all_apps/specified_application_indices/Random_val_indices/Random_20per_' + app_single_name + '_y_val.npy')
if selection_type == 'C':
path = glob.glob('/Users/yzamora/power/nvidia_gpus/all_apps/specified_application_indices/Random_val_indices/*')
else:
elif selection_type == "DHFULL":
model_path = "/Users/yzamora/power/nvidia_gpus/all_apps/deephyper_final_models/fullsetdata_70-VPOST_best"
else:
if selection_type in ['C', 'CAL']:
type_c = 'unique'
else:
type_c = 'best'
model_path = '/Users/yzamora/power/nvidia_gpus/all_apps/deephyper_final_models/' + selection_type + '_one_20-POST_' + type_c
# When using .sav models
model = None
if selection_type in ["RF", "RFAL", "RFFULL"]:
model = pickle.load(open(model_path + '.sav', 'rb'))
elif selection_type != 'OldNew':
model = tf.keras.models.load_model(
model_path + '.h5', custom_objects={'r2': selectMetric('r2')})
if selection_type in ['RFFULL', 'CFFULL', 'DHFULL', 'OldNew']:
path = glob.glob('/Users/yzamora/Desktop/fullset_specified_apps/*.npy')
elif selection_type in ['C', 'OldNew', 'RF']:
path = glob.glob(
'/Users/yzamora/power/nvidia_gpus/all_apps/specified_application_indices/Random_val_indices/Random'
+ '*.npy')
elif selection_type in ['RFAL', 'CAL', 'AL']:
path = glob.glob(
'/Users/yzamora/power/nvidia_gpus/all_apps/specified_application_indices/AL_var_val/AL*.npy'
)
else:
path = glob.glob(
'/Users/yzamora/power/nvidia_gpus/all_apps/specified_application_indices/'
+ selection_type + '_val_indices/' + selection_type + '*.npy')
loss_weight[96] = 1#17
loss_weight[33] = 100#15 dram_read
loss_weight[34] = 100#14 dram_Write
loss_weight[105] = 1#13
loss_weight[106] = 1#10
#model = load_model('bslh_DL_wbias.h5',custom_objects={'loss': weighted_mse(loss_weight)}) # first attempt
#model = load_model('20perdata_dram_bias.h5',custom_objects={'loss':weighted_mse(loss_weight)})
model = load_model('12deeper-noappweight_membounds2_dram_kmeans_zerobias.h5',custom_objects={'loss':weighted_mse(loss_weight)})
"""
##testing deephyper returned model
##model_path = '/Users/yzamora/power/nvidia_gpus/all_apps/deephyper_models/best_model_18_per20.h5'
model_path = '/Users/yzamora/power/nvidia_gpus/all_apps/best_deephyper_mls/AL_one_20-POST_best.h5'
model = tf.keras.models.load_model(model_path,
custom_objects={'r2': selectMetric('r2')})
#model.summary()
df_plot = df_joined[df_joined['memory_bound_V100'] == 1].copy()
colors = ("black", "red", "green", "blue", "pink", "cyan", "yellow")
groups = ('backprop', 'hybridsort', 'kmeans', 'srad', 'stream', 'gaussian',
'leukocyte')
##df_plot = df_joined[df_joined['memory_bound_V100'] == 1].copy()
df_plot = df_joined.copy()
fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(15, 10))
for color, group in zip(colors, groups):
df_predict = df_plot[df_plot['application_name_V100'] == group].copy()