if (X0_eventnum.shape[0] != X0.shape[0]): print( 'problem when loading, #eventnumbers does not match #events, exit') exit(1) else: # some samples won't have an eventnumber, # but ATLAS ones should, since we use it to propagate the weight to reco-level events if (DEBUG): print("No eventnumber found in dataset.") # load model and evaluate weights: carl = RatioEstimator() if (DEBUG): print('Loading model from:', model_out_path) carl.load(model_out_path + '/carl/') r_hat, s_hat = carl.evaluate(X0) # prevent -ve weights (should be rounding only): r_hat = ensure_positive_weight(r_hat) # prevent 0-division r_hat = force_nonzero(r_hat, zero_w_bound) weights = 1. / r_hat # ensure <weights>=1 after cropping weights = weights * len(weights) / weights.sum() maxweight = -1 if (crop_weight_perc > 0): weights, _maxweight = crop_weight_nperc(weights, crop_weight_perc) maxweight = max(maxweight, _maxweight)
logger.info( " Doing evaluation of model trained with datasets: %s , generator variation: %s with %s events.", sample, var, n) else: logger.info( " No datasets available for evaluation of model trained with datasets: %s , generator variation: %s with %s events.", sample, var, n) logger.info("ABORTING") sys.exit() loading = Loader() carl = RatioEstimator() carl.load('models/' + sample + '/' + var + '_carl_' + str(n)) evaluate = ['train', 'val'] for i in evaluate: r_hat, _ = carl.evaluate(x='data/' + sample + '/' + var + '/X0_' + i + '_' + str(n) + '.npy') w = 1. / r_hat loading.load_result( x0='data/' + sample + '/' + var + '/X0_' + i + '_' + str(n) + '.npy', x1='data/' + sample + '/' + var + '/X1_' + i + '_' + str(n) + '.npy', weights=w, label=i, do=sample, var=var, plot=True, n=n, path=p, ) carl.evaluate_performance( x='data/' + sample + '/' + var + '/X_val_' + str(n) + '.npy', y='data/' + sample + '/' + var + '/y_val_' + str(n) + '.npy')
logger.info( " No datasets available for evaluation of model trained with datasets: [{},{}] with {} events." .format(nominal, variation, n)) logger.info("ABORTING") sys.exit() loading = Loader() carl = RatioEstimator() if model: carl.load(model) else: carl.load('models/' + global_name + '_carl_' + str(n)) evaluate = ['train', 'val'] for i in evaluate: print("<evaluate.py::__init__>:: Running evaluation for {}".format(i)) r_hat, s_hat = carl.evaluate(x='data/' + global_name + '/X0_' + i + '_' + str(n) + '.npy') print("s_hat = {}".format(s_hat)) print("r_hat = {}".format(r_hat)) w = 1. / r_hat # I thought r_hat = p_{1}(x) / p_{0}(x) ??? print("w = {}".format(w)) print("<evaluate.py::__init__>:: Loading Result for {}".format(i)) loading.load_result( x0='data/' + global_name + '/X0_' + i + '_' + str(n) + '.npy', x1='data/' + global_name + '/X1_' + i + '_' + str(n) + '.npy', w0='data/' + global_name + '/w0_' + i + '_' + str(n) + '.npy', w1='data/' + global_name + '/w1_' + i + '_' + str(n) + '.npy', metaData='data/' + global_name + '/metaData_' + str(n) + '.pkl', weights=w, features=features, #weightFeature=weightFeature, label=i,
#get the weight from carl-torch (weightCT) evaluated on the same model used for carlAthena and the root file from carlAthena eventVarsCT = ['Njets', 'MET'] eventVarsCA = ['Njets', 'MET', 'weight'] jetVars = ['Jet_Pt', 'Jet_Mass'] lepVars = ['Lepton_Pt'] xCT, _ = load(f=p + '/test.root', events=eventVarsCT, jets=jetVars, leps=lepVars, n=int(n), t='Tree', do=sample) xCT = xCT[sorted(xCT.columns)] carl = RatioEstimator() carl.load('models/' + sample + '/' + var + '_carl_2000001') r_hat, s_hat = carl.evaluate(x=xCT.to_numpy()) weightCT = 1. / r_hat ###carlAthena inference### #load sample with weight infered from carlAthena xCA, _ = load(f=p + '/test.root', events=eventVarsCA, jets=jetVars, leps=lepVars, n=int(n), t='Tree') weightCA = xCA.weight ###compare weights### # draw histograms comparing weight from carl-torch (weightCT) from weight infered through carlAthena (ca.weight) loading = Loader()
# alternatively: crop X% of largest weight crop_weight_perc = -1. #----------------------------------------------------------------------------- if not os.path.exists(out_csv_dir): os.makedirs(out_csv_dir) carl = RatioEstimator() carl.load(model_out_path + '/carl/') evaluate = ['train', 'val'] for i in evaluate: x0 = data_out_path + '/X0_' + i + '.npy' r_hat, s_hat = carl.evaluate(x=data_out_path + '/X0_' + i + '.npy') ## print('what is Carl returning?') ## r=r_hat[0] ## s=s_hat[0] ## print('r=p0/p1,s=p0/(p0+p1)') ## print(r,s,r/(1+r)) ## print('r=p1/p0,s=p0/(p0+p1)') # this ## print(r,s,1/(1+r)) ## print('r=p0/p1,s=p1/(p0+p1)') # this ## print(r,s,1/(1+r)) ## print('r=p1/p0,s=p1/(p0+p1)') ## print(r,s,r/(1+r)) ## ## exit(0) # Carl is returning this: # s_hat = p1 / (p0+p1), where 0 corresponds to X0