def calculate_normalised_xsection(inputs, bin_widths, normalise_to_one=False, covariance_matrix=None, inputMC_covariance_matrix=None): """ Calculates normalised average x-section for each bin: 1/N *1/bin_width sigma_i There are two ways to calculate this 1) N = sum(sigma_i) 2) N = sum(sigma_i/bin_width) The latter one will normalise the total distribution to 1 @param inputs: list of value-error pairs @param bin_widths: bin widths of the inputs """ values = [u.ufloat(i[0], i[1]) for i in inputs] norm_cov_matrix = None norm_corr_matrix = None inputMC_norm_cov_matrix = None if not covariance_matrix is None and not normalise_to_one: values_correlated = u.correlated_values( [v.nominal_value for v in values], covariance_matrix.tolist()) norm = sum(values_correlated) norm_values_correlated = [] # Loop over unfolded number of events with correlated uncertainties # And corresponding bin width # Calculate normalised cross section, with correctly correlated uncertainty for v, width in zip(values_correlated, bin_widths): norm_values_correlated.append(v / width / norm) # Get covariance and correlation matrix for normalised cross section norm_cov_matrix = matrix(u.covariance_matrix(norm_values_correlated)) norm_corr_matrix = matrix(u.correlation_matrix(norm_values_correlated)) result = [(v.nominal_value, v.std_dev) for v in norm_values_correlated] # Get Covariance Matrix for input MC if not inputMC_covariance_matrix is None: inputMC_values_correlated = u.correlated_values( [v.nominal_value for v in values], inputMC_covariance_matrix.tolist()) inputMC_norm = sum(inputMC_values_correlated) inputMC_norm_values_correlated = [] for v, width in zip(inputMC_values_correlated, bin_widths): inputMC_norm_values_correlated.append(v / width / inputMC_norm) inputMC_norm_cov_matrix = matrix( u.covariance_matrix(inputMC_norm_values_correlated)) else: normalisation = 0 if normalise_to_one: normalisation = sum([ value / bin_width for value, bin_width in zip(values, bin_widths) ]) else: normalisation = sum(values) xsections = [(1 / bin_width) * value / normalisation for value, bin_width in zip(values, bin_widths)] result = [(xsection.nominal_value, xsection.std_dev) for xsection in xsections] return result, norm_cov_matrix, norm_corr_matrix, inputMC_norm_cov_matrix
def test_correlated_values(): "Correlated variables." u = uncertainties.ufloat((1, 0.1)) cov = uncertainties.covariance_matrix([u]) # "1" is used instead of u.nominal_value because # u.nominal_value might return a float. The idea is to force # the new variable u2 to be defined through an integer nominal # value: u2, = uncertainties.correlated_values([1], cov) expr = 2 * u2 # Calculations with u2 should be possible, like with u #################### # Covariances between output and input variables: x = ufloat((1, 0.1)) y = ufloat((2, 0.3)) z = -3 * x + y covs = uncertainties.covariance_matrix([x, y, z]) # "Inversion" of the covariance matrix: creation of new # variables: (x_new, y_new, z_new) = uncertainties.correlated_values( [x.nominal_value, y.nominal_value, z.nominal_value], covs, tags=["x", "y", "z"] ) # Even the uncertainties should be correctly reconstructed: assert matrices_close(numpy.array((x, y, z)), numpy.array((x_new, y_new, z_new))) # ... and the covariances too: assert matrices_close(numpy.array(covs), numpy.array(uncertainties.covariance_matrix([x_new, y_new, z_new]))) assert matrices_close(numpy.array([z_new]), numpy.array([-3 * x_new + y_new])) #################### # ... as well as functional relations: u = ufloat((1, 0.05)) v = ufloat((10, 0.1)) sum_value = u + 2 * v # Covariance matrices: cov_matrix = uncertainties.covariance_matrix([u, v, sum_value]) # Correlated variables can be constructed from a covariance matrix, if # NumPy is available: (u2, v2, sum2) = uncertainties.correlated_values([x.nominal_value for x in [u, v, sum_value]], cov_matrix) # matrices_close() is used instead of _numbers_close() because # it compares uncertainties too: assert matrices_close(numpy.array([0]), numpy.array([sum2 - (u2 + 2 * v2)]))
def calculate_xsection(inputs, bin_widths, luminosity, efficiency=1., covariance_matrix=None, inputMC_covariance_matrix=None): ''' BUG: this doesn't work unless the inputs are unfolded! inputs = list of value-error pairs luminosity = integrated luminosity of the measurement ''' abs_cov_matrix = None abs_corr_matrix = None inputMC_abs_cov_matrix = None result = [] add_result = result.append values = [u.ufloat(i[0], i[1]) for i in inputs] if not covariance_matrix is None: values_correlated = u.correlated_values( [v.nominal_value for v in values], covariance_matrix.tolist()) abs_values_correlated = [] # Loop over unfolded number of events with correlated uncertainties # And corresponding bin width # Calculate absolute cross section, with correctly correlated uncertainty for v, width in zip(values_correlated, bin_widths): abs_values_correlated.append(v / width / luminosity / efficiency) # Get covariance and correlation matrix for absolute cross section abs_cov_matrix = matrix(u.covariance_matrix(abs_values_correlated)) abs_corr_matrix = matrix(u.correlation_matrix(abs_values_correlated)) result = [(v.nominal_value, v.std_dev) for v in abs_values_correlated] # Get Covariance Matrix for input MC if not inputMC_covariance_matrix is None: inputMC_values_correlated = u.correlated_values( [v.nominal_value for v in values], inputMC_covariance_matrix.tolist()) inputMC_abs_values_correlated = [] for v, width in zip(inputMC_values_correlated, bin_widths): inputMC_abs_values_correlated.append(v / width / luminosity / efficiency) inputMC_abs_cov_matrix = matrix( u.covariance_matrix(inputMC_abs_values_correlated)) else: for valueAndErrors, binWidth in zip(inputs, bin_widths): value = valueAndErrors[0] error = valueAndErrors[1] xsection = value / (luminosity * efficiency * binWidth) xsection_error = error / (luminosity * efficiency * binWidth) add_result((xsection, xsection_error)) return result, abs_cov_matrix, abs_corr_matrix, inputMC_abs_cov_matrix
def test_copy(): "Standard copy module integration" import gc x = ufloat((3, 0.1)) assert x == x y = copy.copy(x) assert x != y assert not (x == y) assert y in list( y.derivatives.keys()) # y must not copy the dependence on x z = copy.deepcopy(x) assert x != z # Copy tests on expressions: t = x + 2 * z # t depends on x: assert x in t.derivatives # The relationship between the copy of an expression and the # original variables should be preserved: t_copy = copy.copy(t) # Shallow copy: the variables on which t depends are not copied: assert x in t_copy.derivatives assert (uncertainties.covariance_matrix( [t, z]) == uncertainties.covariance_matrix([t_copy, z])) # However, the relationship between a deep copy and the original # variables should be broken, since the deep copy created new, # independent variables: t_deepcopy = copy.deepcopy(t) assert x not in t_deepcopy.derivatives assert (uncertainties.covariance_matrix([t, z]) != uncertainties.covariance_matrix([t_deepcopy, z])) # Test of implementations with weak references: # Weak references: destroying a variable should never destroy the # integrity of its copies (which would happen if the copy keeps a # weak reference to the original, in its derivatives member: the # weak reference to the original would become invalid): del x gc.collect() assert y in list(y.derivatives.keys())
def calculation_Q_conventional(K, a): if kontrola_rozmeru(K, a)==False: return False, False, False Q = -np.dot(K, a) Q_covarianceMatrix = covariance_matrix(Q) Q_correlationMatrix = correlation_matrix(Q) return Q, Q_covarianceMatrix, Q_correlationMatrix
def test_correlated_values_correlation_mat(): ''' Tests the input of correlated value. Test through their correlation matrix (instead of the covariance matrix). ''' x = ufloat((1, 0.1)) y = ufloat((2, 0.3)) z = -3 * x + y cov_mat = uncertainties.covariance_matrix([x, y, z]) std_devs = numpy.sqrt(numpy.array(cov_mat).diagonal()) corr_mat = cov_mat / std_devs / std_devs[numpy.newaxis].T # We make sure that the correlation matrix is indeed diagonal: assert (corr_mat - corr_mat.T).max() <= 1e-15 # We make sure that there are indeed ones on the diagonal: assert (corr_mat.diagonal() - 1).max() <= 1e-15 # We try to recover the correlated variables through the # correlation matrix (not through the covariance matrix): nominal_values = [v.nominal_value for v in (x, y, z)] std_devs = [v.std_dev() for v in (x, y, z)] x2, y2, z2 = uncertainties.correlated_values_norm( list(zip(nominal_values, std_devs)), corr_mat) # matrices_close() is used instead of _numbers_close() because # it compares uncertainties too: # Test of individual variables: assert matrices_close(numpy.array([x]), numpy.array([x2])) assert matrices_close(numpy.array([y]), numpy.array([y2])) assert matrices_close(numpy.array([z]), numpy.array([z2])) # Partial correlation test: assert matrices_close(numpy.array([0]), numpy.array([z2 - (-3 * x2 + y2)])) # Test of the full covariance matrix: assert matrices_close( numpy.array(cov_mat), numpy.array(uncertainties.covariance_matrix([x2, y2, z2])))
def test_copy(): "Standard copy module integration" import gc x = ufloat((3, 0.1)) assert x == x y = copy.copy(x) assert x != y assert not(x == y) assert y in y.derivatives.keys() # y must not copy the dependence on x z = copy.deepcopy(x) assert x != z # Copy tests on expressions: t = x + 2*z # t depends on x: assert x in t.derivatives # The relationship between the copy of an expression and the # original variables should be preserved: t_copy = copy.copy(t) # Shallow copy: the variables on which t depends are not copied: assert x in t_copy.derivatives assert (uncertainties.covariance_matrix([t, z]) == uncertainties.covariance_matrix([t_copy, z])) # However, the relationship between a deep copy and the original # variables should be broken, since the deep copy created new, # independent variables: t_deepcopy = copy.deepcopy(t) assert x not in t_deepcopy.derivatives assert (uncertainties.covariance_matrix([t, z]) != uncertainties.covariance_matrix([t_deepcopy, z])) # Test of implementations with weak references: # Weak references: destroying a variable should never destroy the # integrity of its copies (which would happen if the copy keeps a # weak reference to the original, in its derivatives member: the # weak reference to the original would become invalid): del x gc.collect() assert y in y.derivatives.keys()
def test_correlated_values_correlation_mat(): ''' Tests the input of correlated value. Test through their correlation matrix (instead of the covariance matrix). ''' x = ufloat((1, 0.1)) y = ufloat((2, 0.3)) z = -3*x+y cov_mat = uncertainties.covariance_matrix([x, y, z]) std_devs = numpy.sqrt(numpy.array(cov_mat).diagonal()) corr_mat = cov_mat/std_devs/std_devs[numpy.newaxis].T # We make sure that the correlation matrix is indeed diagonal: assert (corr_mat-corr_mat.T).max() <= 1e-15 # We make sure that there are indeed ones on the diagonal: assert (corr_mat.diagonal()-1).max() <= 1e-15 # We try to recover the correlated variables through the # correlation matrix (not through the covariance matrix): nominal_values = [v.nominal_value for v in (x, y, z)] std_devs = [v.std_dev() for v in (x, y, z)] x2, y2, z2 = uncertainties.correlated_values_norm( list(zip(nominal_values, std_devs)), corr_mat) # matrices_close() is used instead of _numbers_close() because # it compares uncertainties too: # Test of individual variables: assert matrices_close(numpy.array([x]), numpy.array([x2])) assert matrices_close(numpy.array([y]), numpy.array([y2])) assert matrices_close(numpy.array([z]), numpy.array([z2])) # Partial correlation test: assert matrices_close(numpy.array([0]), numpy.array([z2-(-3*x2+y2)])) # Test of the full covariance matrix: assert matrices_close( numpy.array(cov_mat), numpy.array(uncertainties.covariance_matrix([x2, y2, z2])))
def calculation_Q_conventional(K, a_out, a): #pridani vnejsi koncentrace do vektoru koncentraci a = np.append(a, ufloat(a_out, 0.05 * a_out)) if kontrola_rozmeru(K, a) == False: return False, False, False Q = -np.dot(K, a) Q_covarianceMatrix = covariance_matrix(Q) Q_correlationMatrix = correlation_matrix(Q) return Q, Q_covarianceMatrix, Q_correlationMatrix
def calculation_Q_conventional(): K, a = prepare_for_calculation() # breakpoint() if np.isscalar(K): print('Nelze provest vypocet.') Q = -np.dot(K, a) Q_covarianceMatrix = covariance_matrix(Q) Q_correlationMatrix = correlation_matrix(Q) return Q, Q_covarianceMatrix, Q_correlationMatrix
def linleastsquares(functions, x_values, y_values): y = unp.nominal_values(y_values) Z = np.linalg.inv(unc.covariance_matrix(y_values)) A = np.column_stack([f(x_values) for f in functions]) invATA = np.linalg.inv(A.T @ Z @ A) params = invATA @ A.T @ Z @ y cov = invATA return unc.correlated_values(params.flat, cov)
def test_covariances(): "Covariance matrix" x = ufloat((1, 0.1)) y = -2 * x + 10 z = -3 * x covs = uncertainties.covariance_matrix([x, y, z]) # Diagonal elements are simple: assert _numbers_close(covs[0][0], 0.01) assert _numbers_close(covs[1][1], 0.04) assert _numbers_close(covs[2][2], 0.09) # Non-diagonal elements: assert _numbers_close(covs[0][1], -0.02)
def test_covariances(): "Covariance matrix" x = ufloat((1, 0.1)) y = -2*x+10 z = -3*x covs = uncertainties.covariance_matrix([x, y, z]) # Diagonal elements are simple: assert _numbers_close(covs[0][0], 0.01) assert _numbers_close(covs[1][1], 0.04) assert _numbers_close(covs[2][2], 0.09) # Non-diagonal elements: assert _numbers_close(covs[0][1], -0.02)
def linleastsquares(functionlist, x_values, y_values): y = np.matrix(unp.nominal_values(y_values)).T Z = np.matrix(unc.covariance_matrix(y_values)).I # N rows vor every value, p columns for every function dim = (len(x_values), len(functionlist)) A = np.matrix(np.zeros(dim)) for i, func in enumerate(functionlist): A[:, i] = func(x_values)[:, np.newaxis] invATA = (A.T * Z * A).I params = invATA * A.T * Z * y cov = invATA return (np.array(unc.correlated_values(params.flat, np.array(cov))))
def linleastsquares(functionlist, x_values, y_values): y = np.matrix(unp.nominal_values(y_values)).T Z = np.matrix(unc.covariance_matrix(y_values)).I # N rows vor every value, p columns for every function dim = (len(x_values), len(functionlist)) A = np.matrix(np.zeros(dim)) for i, func in enumerate(functionlist): A[:, i] = func(x_values)[:, np.newaxis] invATA = (A.T * Z * A).I params = invATA * A.T * Z * y cov = invATA return (np.array(unc.correlated_values(params.flat, np.array(cov))))
def calibration(detector_name, mu, s_mu): """ does a linear fit fit three points !!! """ def lin_func(x, a, b): return(a*x + b) fit_, cov_fit_ = curve_fit(lin_func, Es, mu, p0=None, sigma=s_mu) fit_corr_ = uc.correlated_values(fit_, cov_fit_) fit_corr = np.array([1 / fit_corr_[0], - fit_corr_[1] / fit_corr_[0]]) fit = un.nominal_values(fit_corr) cov_fit = uc.covariance_matrix(fit_corr) s_fit = un.std_devs(fit_corr) fit_r, s_fit_r = err_round(fit, cov_fit) fit_both = np.array([fit_r, s_fit_r]) fit_both = np.reshape(fit_both.T, np.size(fit_both)) fig1, ax1 = plt.subplots(1, 1) if not save_fig: fig1.suptitle("Calibration: " + detector_name) #plot1, = ax1.plot(mu, Es, '.', alpha=0.9) mu_grid = np.linspace(0, 700, 100) plot_fit, = ax1.plot(mu_grid, lin_func(mu_grid, *fit), alpha=0.5) ax1.errorbar(mu, Es, xerr=s_mu, fmt='.', alpha=0.99, c=plot_fit.get_color()) # errors of t are not changed! ax1.set_xlabel("channel") ax1.set_ylabel("Energy / keV") textstr = 'Results of linear fit for %s'%detector_name + ":\n" textstr += '\\begin{eqnarray*}\ E(\mu)&=& a \mu + E_0 \\\\ \ a &=& (%.4f \pm %.4f)\, \mathrm{keV / ch} \\\\ \ E_0 &=& (%.1f \pm %.1f)\, \mathrm{keV} \\\\ \ \end{eqnarray*}'%tuple(fit_both) ax1.text(0.1, 0.95, textstr, transform=ax1.transAxes, va='top', bbox=props) if show_fig: fig1.show() if save_fig: file_name = "detector_calibration_" + detector_name fig1.savefig(fig_dir + file_name + ".pdf") fig1.savefig(fig_dir + file_name + ".png") return fit_corr[0]
def ucurve_fit(f, xdata, ydata, **kwargs): """ Wrapper for curve_fit that allows use of uncertainties both in model and data. It returns a tuple o ufloat for parametres, correlated. """ x, ux = unpack_unarray(xdata) y, uy = unpack_unarray(ydata) uy = unc.covariance_matrix( ydata ) # I use the covariance matrix as sigma, to fit correlated values @wraps(f ) # need for pass the number of parametres. Without curve_fit fails def model(x, *pars): # if you give to nominal_values a standard numpy.array it returns it return unumpy.nominal_values(f(x, *pars)) p, cov = curve_fit(f=model, xdata=x, ydata=y, sigma=uy, **kwargs) return unc.correlated_values(p, cov)
def weighted_mean(y, covy=None): """ Weighted mean (with covariance matrix). Parameters ---------- y : array of numbers or ufloats covy : None or matrix If covy is None, y must be an array of ufloats. Returns ------- a : ufloat Weighted mean of y. Q : float Chisquare (the value of the minimized quadratic form at the minimum). """ # get covariance matrix if covy is None: covy = un.covariance_matrix(y) else: y = un.correlated_values(y, covy) # compute weighted average inv_covy = np.linalg.inv(covy) vara = 1 / np.sum(inv_covy) a = vara * np.sum(np.dot(inv_covy, y)) # check computation of uncertainties module against direct computation assert np.allclose(vara, a.s ** 2) # compute chisquare res = unp.nominal_values(y) - a.n Q = float(res.reshape(1,-1) @ inv_covy @ res.reshape(-1,1)) return a, Q
def formatcov(x, cov=None, labels=None, corrfmt='.0f'): """ Format an estimate with a covariance matrix as an upper triangular matrix with values on the diagonal (with uncertainties) and correlations off-diagonal. Parameters ---------- x : M-length array Values to be written on the diagonal. cov : (M, M) matrix or None Covariance matrix from which uncertainties and correlations are computed. If None, a covariance matrix is extracted from x with uncertainties.covariance_matrix(x). labels : list of strings Labels for the header of the matrix. If there are less than M labels, only the first elements are given labels. corrfmt : str Format for the correlations. Returns ------- matrix : TextMatrix A TextMatrix instance. Can be converted to a string with str(). Has a method latex() to format as a LaTeX table. See also -------- TextMatrix Examples -------- >>> popt, pcov = scipy.optimize.curve_fit(f, x, y, ...) >>> print(formatcov(popt, pcov)) >>> print(formatcov(popt, pcov).latex()) # LaTeX table """ if cov is None: cov = uncertainties.covariance_matrix(x) x = unumpy.nominal_values(x) if isinstance(x, dict) and isinstance(cov, dict): keys = list(x.keys()) x = [x[key] for key in keys] cov = [[float(cov[keyi, keyj]) for keyj in keys] for keyi in keys] if labels is None: labels = keys pars = _uformat_vect(x, np.sqrt(np.diag(cov))) corr = normcov(cov) * 100 matrix = [] if not (labels is None): if len(labels) < len(x): labels = list(labels) + [''] * (len(x) - len(labels)) elif len(labels) > len(x): labels = labels[:len(x)] matrix.append(labels) for i in range(len(corr)): matrix.append([pars[i]]) for j in range(i + 1, len(corr)): c = corr[i, j] cs = ('{:' + corrfmt + '} %').format(c) if math.isfinite(c) else str(c) matrix[-1].append(cs) return TextMatrix(matrix, fill_side='left')
def calculate(self, dataframe, ibin): # >>> acceptance selection var0 = [True,]*len(dataframe) if self.delR: var0 = var0 & (dataframe['delR'] > self.delR) if self.delDxy: var0 = var0 & (dataframe['delDxy'] < self.delDxy) if self.delDz: var0 = var0 & (dataframe['delDz'] < self.delDz) if all(var0): df = dataframe.copy() else: df = dataframe.loc[var0].copy() # <<< acceptance selection # >>> columns for tnp efficiency df['pass_muon'] = df['muon_ID'] >= self.ID df['pass_antiMuon'] = df['antiMuon_ID'] >= self.ID if self.pfIso: df['pass_muon'] = df['pass_muon'] & (df['muon_pfIso'] < self.pfIso) df['pass_antiMuon'] = df['pass_antiMuon'] & (df['antiMuon_pfIso'] < self.pfIso) if self.tkIso: df['pass_muon'] = df['pass_muon'] & (df['muon_tkIso'] < self.tkIso) df['pass_antiMuon'] = df['pass_antiMuon'] & (df['antiMuon_tkIso'] < self.tkIso) df['pass_muon_antiMuon'] = df['pass_muon'] & df['pass_antiMuon'] nMinus = df['pass_muon'].sum() nPlus = df['pass_antiMuon'].sum() nPlusMinus = df['pass_muon_antiMuon'].sum() if self.HLT: df['pass_muon_HLT'] = df['pass_muon_antiMuon'] & df['muon_hlt_{0}'.format(self.HLT)] df['pass_antiMuon_HLT'] = df['pass_muon_antiMuon'] & df['antiMuon_hlt_{0}'.format(self.HLT)] df['pass_muon_antiMuon_HLT'] = df['pass_muon_HLT'] & df['pass_antiMuon_HLT'] nMinus_HLT = df['pass_muon_HLT'].sum() nPlus_HLT = df['pass_antiMuon_HLT'].sum() nPlusMinus_HLT = df['pass_muon_antiMuon_HLT'].sum() # <<< columns for tnp efficiency # >>> columns for true efficiency if self.HLT: df['reco'] = df['pass_muon_antiMuon'] & (df['pass_muon_HLT'] | df['pass_antiMuon_HLT']) else: df['reco'] = df['pass_muon_antiMuon'] df['not_reco'] = ~df['reco'] nZReco = df['reco'].sum() nZNotReco = df['not_reco'].sum() # <<< columns for true efficiency # >>> construct full covariance matrix if self.HLT: corr_matrix = df[['reco','not_reco','pass_muon_antiMuon','pass_muon','pass_antiMuon','pass_muon_antiMuon_HLT','pass_muon_HLT','pass_antiMuon_HLT']].corr() nZReco, nZNotReco, nPlusMinus, nMinus, nPlus, nPlusMinus_HLT, nMinus_HLT, nPlus_HLT = unc.correlated_values_norm( [(nZReco, np.sqrt(nZReco)), (nZNotReco, np.sqrt(nZNotReco)), (nPlusMinus, np.sqrt(nPlusMinus)), (nMinus, np.sqrt(nMinus)), (nPlus, np.sqrt(nPlus)), (nPlusMinus_HLT, np.sqrt(nPlusMinus_HLT)), (nMinus_HLT, np.sqrt(nMinus_HLT)), (nPlus_HLT, np.sqrt(nPlus_HLT))], corr_matrix) else: corr_matrix = df[['reco','not_reco','pass_muon_antiMuon','pass_muon','pass_antiMuon']].corr() nZReco, nZNotReco, nPlusMinus, nMinus, nPlus = unc.correlated_values_norm( [(nZReco, np.sqrt(nZReco)), (nZNotReco, np.sqrt(nZNotReco)), (nPlusMinus, np.sqrt(nPlusMinus)), (nMinus, np.sqrt(nMinus)), (nPlus, np.sqrt(nPlus))], corr_matrix) # <<< construct full covariance matrix # >>> compute true and tnp efficiency eff_true = nZReco / (nZNotReco + nZReco) MuPlusEff = nPlusMinus/nMinus MuMinusEff = nPlusMinus/nPlus eff_tnp = MuPlusEff * MuMinusEff if self.HLT: MuPlusEff_HLT = nPlusMinus_HLT/nMinus_HLT MuMinusEff_HLT = nPlusMinus_HLT/nPlus_HLT eff_tnpZ = (1 - (1 - MuPlusEff_HLT) * (1 - MuMinusEff_HLT) ) * eff_tnp # <<< compute true and tnp efficiency else: eff_tnpZ = eff_tnp # >>> store self.eff_corr.append(unc.covariance_matrix([eff_tnpZ, eff_true])) self.eff_tnp.append(eff_tnp) self.eff_tnpZ.append(eff_tnpZ) self.eff_true.append(eff_true)
#(3.1415+/-0)e+10 #>>> print ufloat(3.1415, 0.0005) #3.1415+/-0.0005 #>>> print '{:.2f}'.format(ufloat(3.14, 0.001)) #3.14+/-0.00 #>>> print '{:.2f}'.format(ufloat(3.14, 0.00)) #3.14+/-0 u = ufloat(1, 0.1, "u variable") # Tag v = ufloat(10, 0.1, "v variable") sum_value = u+2*v print(sum_value) cov_matrix = covariance_matrix([u, v, sum_value]) print(cov_matrix) (u2, v2, sum2) = uncertainties.correlated_values([1, 10, 21], cov_matrix) print(sum2) for (var, error) in sum_value.error_components().items(): print("{}: {}".format(var.tag, error)) output = subprocess.check_output(["echo", "Hello World!"]) print(subprocess.getoutput('ls /bin/ls')) print(subprocess.getoutput('echo %s' %"Hello World!")) print(b'hi'.decode('ascii')) print(output)
def __init__(self, dir_x, dir_y, dir_z, unc_x, unc_y, unc_z, num_samples=1000000, random_seed=42, weighted_normalization=True, fix_delta=True): """Initialize DNN LLH object. Parameters ---------- dir_x : float The best fit direction vector x component. This is the output of the DNN reco for the x-component. dir_y : float The best fit direction vector y component. This is the output of the DNN reco for the y-component. dir_z : float The best fit direction vector z component. This is the output of the DNN reco for the z-component. unc_x : float The estimated uncertainty for the direction vector x component. This is the output of the DNN reco for the estimated uncertainty. unc_y : float The estimated uncertainty for the direction vector y component. This is the output of the DNN reco for the estimated uncertainty. unc_z : float The estimated uncertainty for the direction vector z component. This is the output of the DNN reco for the estimated uncertainty. num_samples : int, optional Number of samples to sample for internal calculations. The more samples, the more accurate, but also slower. random_seed : int, optional Random seed for sampling. weighted_normalization : bool, optional If True the normalization vectors get normalized according to the uncertainty on each of its components. If False, the vectors get scaled by their norm to obtain unit vectors. fix_delta : bool, optional If True, the sampled direction vectors will sampled in a way such that the deltas of the angles: abs(azimuth - sampled_azimuth) and abs(zenith - sampled_zenith) follow the expected distribution. """ self.weighted_normalization = weighted_normalization u_dir_x = ufloat(dir_x, unc_x) u_dir_y = ufloat(dir_y, unc_y) u_dir_z = ufloat(dir_z, unc_z) u_dir_x, u_dir_y, u_dir_z = self.u_normalize_dir( u_dir_x, u_dir_y, u_dir_z) # Assign values with propagated and normalized vector u_zenith, u_azimuth = self.u_get_zenith_azimuth( u_dir_x, u_dir_y, u_dir_z) self.dir_x = u_dir_x.nominal_value self.dir_y = u_dir_y.nominal_value self.dir_z = u_dir_z.nominal_value self.zenith = unumpy.nominal_values(u_zenith) self.azimuth = unumpy.nominal_values(u_azimuth) self.unc_zenith = unumpy.std_devs(u_zenith) self.unc_azimuth = unumpy.std_devs(u_azimuth) cov = np.array(covariance_matrix([u_zenith, u_azimuth])) DNN_LLH_Base_Elliptical.__init__(self, self.zenith, self.azimuth, cov, num_samples, random_seed, weighted_normalization, fix_delta)
def __init__(self, dir_x, dir_y, dir_z, unc_x, unc_y, unc_z, propagate_errors=False, num_samples=1000000, random_seed=42, scale_unc=True, weighted_normalization=True, fix_delta=True): """Initialize DNN LLH object. Parameters ---------- dir_x : float The best fit direction vector x component. This is the output of the DNN reco for the x-component. dir_y : float The best fit direction vector y component. This is the output of the DNN reco for the y-component. dir_z : float The best fit direction vector z component. This is the output of the DNN reco for the z-component. unc_x : float The estimated uncertainty for the direction vector x component. This is the output of the DNN reco for the estimated uncertainty. unc_y : float The estimated uncertainty for the direction vector y component. This is the output of the DNN reco for the estimated uncertainty. unc_z : float The estimated uncertainty for the direction vector z component. This is the output of the DNN reco for the estimated uncertainty. propagate_errors : bool, optional Propagate errors and account for correlations. num_samples : int, optional Number of samples to sample for internal calculations. The more samples, the more accurate, but also slower. random_seed : int, optional Random seed for sampling. scale_unc : bool, optional Due to the normalization of the direction vectors, the components of the vector are correlated, hence the actual spread in sampled direction vectors shrinks. The nn model predicts the Gaussian Likelihood of the normalized vectors (if normalization is included) in network model. In this case, the uncertainties of the direction vector components can be scaled to account for this correlation. If set to True, the uncertainties will be scaled. weighted_normalization : bool, optional If True the normalization vectors get normalized according to the uncertainty on each of its components. If False, the vectors get scaled by their norm to obtain unit vectors. fix_delta : bool, optional If True, the sampled direction vectors will sampled in a way such that the deltas: abs(dir_i - sampled_dir_i) follows the expected distribution. """ # call init from base class DNN_LLH_Base.__init__(self, dir_x, dir_y, dir_z, unc_x, unc_y, unc_z, random_seed, weighted_normalization) self._num_samples = num_samples self.propagate_errors = propagate_errors self._fix_delta = fix_delta if self.propagate_errors: # propagate errors u_dir_x = unumpy.uarray(dir_x, unc_x) u_dir_y = unumpy.uarray(dir_y, unc_y) u_dir_z = unumpy.uarray(dir_z, unc_z) u_dir_x, u_dir_y, u_dir_z = self.u_normalize_dir( u_dir_x, u_dir_y, u_dir_z) # Assign values with propagated and normalized vector self.unc_x = u_dir_x.std_dev self.unc_y = u_dir_y.std_dev self.unc_z = u_dir_z.std_dev self.dir_x = u_dir_x.nominal_value self.dir_y = u_dir_y.nominal_value self.dir_z = u_dir_z.nominal_value self.cov_matrix = np.array( uncertainties.covariance_matrix([u_dir_x, u_dir_y, u_dir_z])) self.dist = multivariate_normal(mean=(self.dir_x, self.dir_y, self.dir_z), cov=self.cov_matrix, allow_singular=True) else: self.unc_x = unc_x self.unc_y = unc_y self.unc_z = unc_z self.dir_x, self.dir_y, self.dir_z = \ self.normalize_dir(dir_x, dir_y, dir_z) # ------------------------- # scale up unc if necessary # ------------------------- self.scale_unc = scale_unc if self.scale_unc: def _scale(): dir_x_s, dir_y_s, dir_z_s = self.sample_dir( min(self._num_samples, 1000)) # print('scaling x by:', self.unc_x / np.std(dir_x_s)) # print('scaling y by:', self.unc_y / np.std(dir_y_s)) # print('scaling z by:', self.unc_z / np.std(dir_z_s)) self.unc_x *= self.unc_x / np.std(dir_x_s) self.unc_y *= self.unc_y / np.std(dir_y_s) self.unc_z *= self.unc_z / np.std(dir_z_s) _scale() # ------------------------- self.zenith, self.azimuth = self.get_zenith_azimuth( self.dir_x, self.dir_y, self.dir_z) # sample contours self.dir_x_s, self.dir_y_s, self.dir_z_s = \ self.sample_dir(self._num_samples) self.neg_llh_values = -self.log_prob_dir(self.dir_x_s, self.dir_y_s, self.dir_z_s) # sort sampled points according to neg llh sorted_indices = np.argsort(self.neg_llh_values) self.dir_x_s = self.dir_x_s[sorted_indices] self.dir_y_s = self.dir_y_s[sorted_indices] self.dir_z_s = self.dir_z_s[sorted_indices] self.neg_llh_values = self.neg_llh_values[sorted_indices] # get sampled zenith and azimuth self.zenith_s, self.azimuth_s = self.get_zenith_azimuth( self.dir_x_s, self.dir_y_s, self.dir_z_s)
def test_correlated_values(): """ Correlated variables. Test through the input of the (full) covariance matrix. """ u = uncertainties.ufloat((1, 0.1)) cov = uncertainties.covariance_matrix([u]) # "1" is used instead of u.nominal_value because # u.nominal_value might return a float. The idea is to force # the new variable u2 to be defined through an integer nominal # value: u2, = uncertainties.correlated_values([1], cov) expr = 2 * u2 # Calculations with u2 should be possible, like with u #################### # Covariances between output and input variables: x = ufloat((1, 0.1)) y = ufloat((2, 0.3)) z = -3 * x + y covs = uncertainties.covariance_matrix([x, y, z]) # Test of the diagonal covariance elements: assert matrices_close(numpy.array([v.std_dev()**2 for v in (x, y, z)]), numpy.array(covs).diagonal()) # "Inversion" of the covariance matrix: creation of new # variables: (x_new, y_new, z_new) = uncertainties.correlated_values( [x.nominal_value, y.nominal_value, z.nominal_value], covs, tags=['x', 'y', 'z']) # Even the uncertainties should be correctly reconstructed: assert matrices_close(numpy.array((x, y, z)), numpy.array((x_new, y_new, z_new))) # ... and the covariances too: assert matrices_close( numpy.array(covs), numpy.array(uncertainties.covariance_matrix([x_new, y_new, z_new]))) assert matrices_close(numpy.array([z_new]), numpy.array([-3 * x_new + y_new])) #################### # ... as well as functional relations: u = ufloat((1, 0.05)) v = ufloat((10, 0.1)) sum_value = u + 2 * v # Covariance matrices: cov_matrix = uncertainties.covariance_matrix([u, v, sum_value]) # Correlated variables can be constructed from a covariance # matrix, if NumPy is available: (u2, v2, sum2) = uncertainties.correlated_values( [x.nominal_value for x in [u, v, sum_value]], cov_matrix) # matrices_close() is used instead of _numbers_close() because # it compares uncertainties too: assert matrices_close(numpy.array([u]), numpy.array([u2])) assert matrices_close(numpy.array([v]), numpy.array([v2])) assert matrices_close(numpy.array([sum_value]), numpy.array([sum2])) assert matrices_close(numpy.array([0]), numpy.array([sum2 - (u2 + 2 * v2)]))
def __init__(self, values, cov=None, stds=None, info=None, labels=None): """ Store parameters and associated uncertainties. Parameters ---------- values : array_like, dict or uncertainties object Either the nominal values of the parameters, a dictionary containing `values` and `stds` or `cov` used to create a parameter object, or values with associated uncertainties created by the `uncertainties` package. cov : array_like The covariance matrix for the specified parameters used for propagating errors. Only used if `values` is array_like, in which case i should be an array of shape (len(values), len(values)). stds : array_like Standard deviations of the values. This does not account for correllated uncertainties - use `cov` for that. Only used if `values` is array_like, in which case `stds` should be the same length as `values`. info : str A description of the parameteters, to be stored alongside the values. labels : array_like Parameter names. """ if isinstance(values, dict): vd = values.copy() values = vd['values'] if 'stds' in vd: stds = vd['stds'] if 'cov' in vd: cov = vd['cov'] if 'info' in vd: info = vd['info'] if 'labels' in vd: labels = vd['labels'] if info is None: info = 'No information given.' self.info = info self.labels = labels self.param_name = '' if ucheck(values): self.values = values else: if cov is not None: self.values = un.correlated_values(values, cov) elif stds is not None: self.values = unp.uarray(values, stds) else: self.values = values if ucheck(self.values): self.nom_values = unp.nominal_values(self.values) self.stds = unp.std_devs(self.values) self.cov = un.covariance_matrix(self.values) else: self.nom_values = np.asanyarray(self.values) self.stds = np.full(len(self.values), np.nan) self.stds = np.full((len(self.values),len(self.values)), np.nan) self.wrap = TextWrapper(linewidth).wrap
def test_monte_carlo_comparison(): """ Full comparison to a Monte-Carlo calculation. Both the nominal values and the covariances are compared between the direct calculation performed in this module and a Monte-Carlo simulation. """ try: import numpy import numpy.random except ImportError: import warnings warnings.warn("Test not performed because NumPy is not available") return # Works on numpy.arrays of Variable objects (whereas umath.sin() # does not): sin_uarrayncert = numpy.vectorize(umath.sin, otypes=[object]) # Example expression (with correlations, and multiple variables combined # in a non-linear way): def function(x, y): """ Function that takes two NumPy arrays of the same size. """ # The uncertainty due to x is about equal to the uncertainty # due to y: return 10 * x**2 - x * sin_uarrayncert(y**3) x = uncertainties.ufloat((0.2, 0.01)) y = uncertainties.ufloat((10, 0.001)) function_result_this_module = function(x, y) nominal_value_this_module = function_result_this_module.nominal_value # Covariances "f*f", "f*x", "f*y": covariances_this_module = numpy.array(uncertainties.covariance_matrix( (x, y, function_result_this_module))) def monte_carlo_calc(n_samples): """ Calculate function(x, y) on n_samples samples and returns the median, and the covariances between (x, y, function(x, y)). """ # Result of a Monte-Carlo simulation: x_samples = numpy.random.normal(x.nominal_value, x.std_dev(), n_samples) y_samples = numpy.random.normal(y.nominal_value, y.std_dev(), n_samples) function_samples = function(x_samples, y_samples) cov_mat = numpy.cov([x_samples, y_samples], function_samples) return (numpy.median(function_samples), cov_mat) (nominal_value_samples, covariances_samples) = monte_carlo_calc(1000000) ## Comparison between both results: # The covariance matrices must be close: # We rely on the fact that covariances_samples very rarely has # null elements: assert numpy.vectorize(test_uncertainties._numbers_close)( covariances_this_module, covariances_samples, 0.05).all(), ( "The covariance matrices do not coincide between" " the Monte-Carlo simulation and the direct calculation:\n" "* Monte-Carlo:\n%s\n* Direct calculation:\n%s" % (covariances_samples, covariances_this_module) ) # The nominal values must be close: assert test_uncertainties._numbers_close( nominal_value_this_module, nominal_value_samples, # The scale of the comparison depends on the standard # deviation: the nominal values can differ by a fraction of # the standard deviation: math.sqrt(covariances_samples[2, 2]) / abs(nominal_value_samples) * 0.5), ( "The nominal value (%f) does not coincide with that of" " the Monte-Carlo simulation (%f), for a standard deviation of %f." % (nominal_value_this_module, nominal_value_samples, math.sqrt(covariances_samples[2, 2])) )
def test_monte_carlo_comparison(): """ Full comparison to a Monte-Carlo calculation. Both the nominal values and the covariances are compared between the direct calculation performed in this module and a Monte-Carlo simulation. """ try: import numpy import numpy.random except ImportError: import warnings warnings.warn("Test not performed because NumPy is not available") return # Works on numpy.arrays of Variable objects (whereas umath.sin() # does not): sin_uarray_uncert = numpy.vectorize(umath.sin, otypes=[object]) # Example expression (with correlations, and multiple variables combined # in a non-linear way): def function(x, y): """ Function that takes two NumPy arrays of the same size. """ # The uncertainty due to x is about equal to the uncertainty # due to y: return 10 * x**2 - x * sin_uarray_uncert(y**3) x = ufloat(0.2, 0.01) y = ufloat(10, 0.001) function_result_this_module = function(x, y) nominal_value_this_module = function_result_this_module.nominal_value # Covariances "f*f", "f*x", "f*y": covariances_this_module = numpy.array( uncertainties.covariance_matrix((x, y, function_result_this_module))) def monte_carlo_calc(n_samples): """ Calculate function(x, y) on n_samples samples and returns the median, and the covariances between (x, y, function(x, y)). """ # Result of a Monte-Carlo simulation: x_samples = numpy.random.normal(x.nominal_value, x.std_dev, n_samples) y_samples = numpy.random.normal(y.nominal_value, y.std_dev, n_samples) # !!! astype() is a temporary fix for NumPy 1.8: function_samples = function(x_samples, y_samples).astype(float) cov_mat = numpy.cov([x_samples, y_samples], function_samples) return (numpy.median(function_samples), cov_mat) (nominal_value_samples, covariances_samples) = monte_carlo_calc(1000000) ## Comparison between both results: # The covariance matrices must be close: # We rely on the fact that covariances_samples very rarely has # null elements: assert numpy.vectorize(test_uncertainties.numbers_close)( covariances_this_module, covariances_samples, 0.05).all(), ( "The covariance matrices do not coincide between" " the Monte-Carlo simulation and the direct calculation:\n" "* Monte-Carlo:\n%s\n* Direct calculation:\n%s" % (covariances_samples, covariances_this_module)) # The nominal values must be close: assert test_uncertainties.numbers_close( nominal_value_this_module, nominal_value_samples, # The scale of the comparison depends on the standard # deviation: the nominal values can differ by a fraction of # the standard deviation: math.sqrt(covariances_samples[2, 2]) / abs(nominal_value_samples) * 0.5), ( "The nominal value (%f) does not coincide with that of" " the Monte-Carlo simulation (%f), for a standard deviation of %f." % (nominal_value_this_module, nominal_value_samples, math.sqrt(covariances_samples[2, 2])))