def evaluate(test_job,
autoencoder,
reg,
lods,
observed_traces,
observed_traces_slices,
out_data,
test_aliases=[]):
# Get the trace(s) for this specific test job
idx = observed_traces_slices[test_job]
trace = observed_traces.a[idx], observed_traces.X[idx], \
observed_traces.Y[idx]
# Extract encoding from given trace(s) and trained autoencoder
extract_and_set_encoding(autoencoder, trace)
proxy = extract_encoding_and_map_to_nearest(autoencoder, trace,
lods.alias_to_id, test_aliases)
out_data['mappings'][test_job].append(proxy)
# Test data (without encodings)
test = lods.test
slices_test = test.slice_by_job_id(lods.alias_to_id)
idxs_test = slices_test[test_job]
X_test = np.hstack([test.a, test.X])[idxs_test, :]
y_test = test.targets.ravel()[idxs_test]
# Test data for regressor prediction (without mapping)
X_test_, y_test_ = translate_X_y(X_test, y_test, autoencoder.centroids)
# Test data for regressor prediction (with mapping)
X_test__, y_test__ = translate_X_y(X_test, y_test,
autoencoder.altered_centroids)
reg_ = reg.clone()
err = reg_.MAPE(X_test_, y_test_)
logging.info("[Test job: {}] \t Error no-opt: {:.2f}".format(
test_job, err))
reg_ = reg.clone()
assert np.abs(err - reg_.MAPE(X_test_, y_test_)) < 1e-10
# Now let's use the data from mapping...
err_mapping = reg_.MAPE(X_test__, y_test__)
logging.info("[Test job: {}] \t Error map: {:.2f}".format(
test_job, err_mapping))
# err, err_mapping
out_data['errs']['no-opt'][test_job].append(err)
out_data['errs']['map'][test_job].append(err_mapping)
def evaluate(test_job,
autoencoder,
reg,
lods,
observed_traces,
observed_traces_slices,
out_data,
test_aliases=[]):
n_knob_cols = len(lods.config['COLS_KNOBS'])
# Get the trace(s) for this specific test job
idx = observed_traces_slices[test_job]
trace = observed_traces.a[idx], observed_traces.X[idx], \
observed_traces.Y[idx]
# Extract encoding from given trace(s) and trained autoencoder
extract_and_set_encoding(autoencoder, trace)
proxy = extract_encoding_and_map_to_nearest(autoencoder,
trace,
lods.alias_to_id,
test_aliases,
within_template=False,
metric='euclidean')
out_data['mappings'][test_job].append(proxy)
# Calibration data for autoencoder
X_calib = np.hstack([observed_traces.a[idx], observed_traces.X[idx]])
y_calib = observed_traces.targets.ravel()[idx]
# Calibration Data for regressor ('without mapping' case):
X_calib_, y_calib_ = translate_X_y(X_calib, y_calib, autoencoder.centroids,
n_knob_cols)
# Calibration Data for regressor ('with mapping' case):
X_calib__, y_calib__ = translate_X_y(X_calib, y_calib,
autoencoder.altered_centroids,
n_knob_cols)
# Test data (without encodings)
test = lods.test
slices_test = test.slice_by_job_id(lods.alias_to_id)
idxs_test = slices_test[test_job]
X_test = np.hstack([test.a, test.X])[idxs_test, :]
y_test = test.targets.ravel()[idxs_test]
# Test data for regressor prediction (without mapping)
X_test_, y_test_ = translate_X_y(X_test, y_test, autoencoder.centroids,
n_knob_cols)
# Test data for regressor prediction (with mapping)
X_test__, y_test__ = translate_X_y(X_test, y_test,
autoencoder.altered_centroids,
n_knob_cols)
reg_ = reg.clone()
err = reg_.MAPE(X_test_, y_test_)
logging.info("[Test job: {}] \t Error no-opt: {:.2f}".format(
test_job, err))
reg_.calibrate(X_calib_, y_calib_)
err_cal = reg_.MAPE(X_test_, y_test_)
logging.info("[Test job: {}] \t Error cal: {:.2f}".format(
test_job, err_cal))
reg_ = reg.clone()
assert np.abs(err - reg_.MAPE(X_test_, y_test_)) < 1e-10
# Now let's use the data from mapping...
err_mapping = reg_.MAPE(X_test__, y_test__)
logging.info("[Test job: {}] \t Error map: {:.2f}".format(
test_job, err_mapping))
reg_.calibrate(X_calib__, y_calib__)
err_mapping_cal = reg_.MAPE(X_test__, y_test__)
logging.info("[Test job: {}] \t Error map_and_cal: {:.2f}".format(
test_job, err_mapping_cal))
# err, err_cal, err_mapping, err_mapping_cal
out_data['errs']['no-opt'][test_job].append(err)
out_data['errs']['cal'][test_job].append(err_cal)
out_data['errs']['map'][test_job].append(err_mapping)
out_data['errs']['map_then_cal'][test_job].append(err_mapping_cal)
