def evaluate(test_job, autoencoder, reg, lods, observed_traces, observed_traces_slices, out_data, test_aliases=[]): # Get the trace(s) for this specific test job idx = observed_traces_slices[test_job] trace = observed_traces.a[idx], observed_traces.X[idx], \ observed_traces.Y[idx] # Extract encoding from given trace(s) and trained autoencoder extract_and_set_encoding(autoencoder, trace) proxy = extract_encoding_and_map_to_nearest(autoencoder, trace, lods.alias_to_id, test_aliases) out_data['mappings'][test_job].append(proxy) # Test data (without encodings) test = lods.test slices_test = test.slice_by_job_id(lods.alias_to_id) idxs_test = slices_test[test_job] X_test = np.hstack([test.a, test.X])[idxs_test, :] y_test = test.targets.ravel()[idxs_test] # Test data for regressor prediction (without mapping) X_test_, y_test_ = translate_X_y(X_test, y_test, autoencoder.centroids) # Test data for regressor prediction (with mapping) X_test__, y_test__ = translate_X_y(X_test, y_test, autoencoder.altered_centroids) reg_ = reg.clone() err = reg_.MAPE(X_test_, y_test_) logging.info("[Test job: {}] \t Error no-opt: {:.2f}".format( test_job, err)) reg_ = reg.clone() assert np.abs(err - reg_.MAPE(X_test_, y_test_)) < 1e-10 # Now let's use the data from mapping... err_mapping = reg_.MAPE(X_test__, y_test__) logging.info("[Test job: {}] \t Error map: {:.2f}".format( test_job, err_mapping)) # err, err_mapping out_data['errs']['no-opt'][test_job].append(err) out_data['errs']['map'][test_job].append(err_mapping)
def evaluate(test_job, autoencoder, reg, lods, observed_traces, observed_traces_slices, out_data, test_aliases=[]): n_knob_cols = len(lods.config['COLS_KNOBS']) # Get the trace(s) for this specific test job idx = observed_traces_slices[test_job] trace = observed_traces.a[idx], observed_traces.X[idx], \ observed_traces.Y[idx] # Extract encoding from given trace(s) and trained autoencoder extract_and_set_encoding(autoencoder, trace) proxy = extract_encoding_and_map_to_nearest(autoencoder, trace, lods.alias_to_id, test_aliases, within_template=False, metric='euclidean') out_data['mappings'][test_job].append(proxy) # Calibration data for autoencoder X_calib = np.hstack([observed_traces.a[idx], observed_traces.X[idx]]) y_calib = observed_traces.targets.ravel()[idx] # Calibration Data for regressor ('without mapping' case): X_calib_, y_calib_ = translate_X_y(X_calib, y_calib, autoencoder.centroids, n_knob_cols) # Calibration Data for regressor ('with mapping' case): X_calib__, y_calib__ = translate_X_y(X_calib, y_calib, autoencoder.altered_centroids, n_knob_cols) # Test data (without encodings) test = lods.test slices_test = test.slice_by_job_id(lods.alias_to_id) idxs_test = slices_test[test_job] X_test = np.hstack([test.a, test.X])[idxs_test, :] y_test = test.targets.ravel()[idxs_test] # Test data for regressor prediction (without mapping) X_test_, y_test_ = translate_X_y(X_test, y_test, autoencoder.centroids, n_knob_cols) # Test data for regressor prediction (with mapping) X_test__, y_test__ = translate_X_y(X_test, y_test, autoencoder.altered_centroids, n_knob_cols) reg_ = reg.clone() err = reg_.MAPE(X_test_, y_test_) logging.info("[Test job: {}] \t Error no-opt: {:.2f}".format( test_job, err)) reg_.calibrate(X_calib_, y_calib_) err_cal = reg_.MAPE(X_test_, y_test_) logging.info("[Test job: {}] \t Error cal: {:.2f}".format( test_job, err_cal)) reg_ = reg.clone() assert np.abs(err - reg_.MAPE(X_test_, y_test_)) < 1e-10 # Now let's use the data from mapping... err_mapping = reg_.MAPE(X_test__, y_test__) logging.info("[Test job: {}] \t Error map: {:.2f}".format( test_job, err_mapping)) reg_.calibrate(X_calib__, y_calib__) err_mapping_cal = reg_.MAPE(X_test__, y_test__) logging.info("[Test job: {}] \t Error map_and_cal: {:.2f}".format( test_job, err_mapping_cal)) # err, err_cal, err_mapping, err_mapping_cal out_data['errs']['no-opt'][test_job].append(err) out_data['errs']['cal'][test_job].append(err_cal) out_data['errs']['map'][test_job].append(err_mapping) out_data['errs']['map_then_cal'][test_job].append(err_mapping_cal)