if (X0_eventnum is not None): if (X0_eventnum.shape[0] != X0.shape[0]): print( 'problem when loading, #eventnumbers does not match #events, exit') exit(1) else: # some samples won't have an eventnumber, # but ATLAS ones should, since we use it to propagate the weight to reco-level events if (DEBUG): print("No eventnumber found in dataset.") # load model and evaluate weights: carl = RatioEstimator() if (DEBUG): print('Loading model from:', model_out_path) carl.load(model_out_path + '/carl/') r_hat, s_hat = carl.evaluate(X0) # prevent -ve weights (should be rounding only): r_hat = ensure_positive_weight(r_hat) # prevent 0-division r_hat = force_nonzero(r_hat, zero_w_bound) weights = 1. / r_hat # ensure <weights>=1 after cropping weights = weights * len(weights) / weights.sum() maxweight = -1 if (crop_weight_perc > 0): weights, _maxweight = crop_weight_nperc(weights, crop_weight_perc) maxweight = max(maxweight, _maxweight)
logger = logging.getLogger(__name__) if os.path.exists('data/' + global_name + '/X_train_' + str(n) + '.npy') and os.path.exists('data/' + global_name + '/metaData_' + str(n) + '.pkl'): logger.info( " Doing calibration of model trained with datasets: [{},{}], with {} events.", nominal, variation, n) else: logger.info( " No datasets available for evaluation of model trained with datasets: [{},{}] with {} events." .format(nominal, variation, n)) logger.info("ABORTING") sys.exit() carl = RatioEstimator() carl.load('models/' + global_name + '_carl_' + str(n)) #load evaluate = ['train'] X = 'data/' + global_name + '/X_train_' + str(n) + '.npy' y = 'data/' + global_name + '/y_train_' + str(n) + '.npy' w = 'data/' + global_name + '/w_train_' + str(n) + '.npy' r_hat, s_hat = carl.evaluate(X) calib = CalibratedClassifier(carl, global_name=global_name) calib.fit(X=X, y=y, w=w) p0, p1, r_cal = calib.predict(X=X) w_cal = 1 / r_cal loading.load_calibration( y_true=y, p1_raw=s_hat, p1_cal=p1, label='calibrated',
logger = logging.getLogger(__name__) if os.path.exists('data/' + sample + '/' + var + '/X_train_' + str(n) + '.npy'): logger.info( " Doing evaluation of model trained with datasets: %s , generator variation: %s with %s events.", sample, var, n) else: logger.info( " No datasets available for evaluation of model trained with datasets: %s , generator variation: %s with %s events.", sample, var, n) logger.info("ABORTING") sys.exit() loading = Loader() carl = RatioEstimator() carl.load('models/' + sample + '/' + var + '_carl_' + str(n)) evaluate = ['train', 'val'] for i in evaluate: r_hat, _ = carl.evaluate(x='data/' + sample + '/' + var + '/X0_' + i + '_' + str(n) + '.npy') w = 1. / r_hat loading.load_result( x0='data/' + sample + '/' + var + '/X0_' + i + '_' + str(n) + '.npy', x1='data/' + sample + '/' + var + '/X1_' + i + '_' + str(n) + '.npy', weights=w, label=i, do=sample, var=var, plot=True, n=n, path=p,
#carl-torch inference### #get the weight from carl-torch (weightCT) evaluated on the same model used for carlAthena and the root file from carlAthena eventVarsCT = ['Njets', 'MET'] eventVarsCA = ['Njets', 'MET', 'weight'] jetVars = ['Jet_Pt', 'Jet_Mass'] lepVars = ['Lepton_Pt'] xCT, _ = load(f=p + '/test.root', events=eventVarsCT, jets=jetVars, leps=lepVars, n=int(n), t='Tree', do=sample) xCT = xCT[sorted(xCT.columns)] carl = RatioEstimator() carl.load('models/' + sample + '/' + var + '_carl_2000001') r_hat, s_hat = carl.evaluate(x=xCT.to_numpy()) weightCT = 1. / r_hat ###carlAthena inference### #load sample with weight infered from carlAthena xCA, _ = load(f=p + '/test.root', events=eventVarsCA, jets=jetVars, leps=lepVars, n=int(n), t='Tree') weightCA = xCA.weight ###compare weights### # draw histograms comparing weight from carl-torch (weightCT) from weight infered through carlAthena (ca.weight)