def __init__(self, modifiers, pdfconfig, builder_data, interpcode='code0', batch_size=None): self.batch_size = batch_size self.interpcode = interpcode assert self.interpcode in ['code0', 'code2', 'code4p'] keys = [f'{mtype}/{m}' for m, mtype in modifiers] histosys_mods = [m for m, _ in modifiers] parfield_shape = ((self.batch_size, pdfconfig.npars) if self.batch_size else (pdfconfig.npars, )) self.param_viewer = ParamViewer(parfield_shape, pdfconfig.par_map, histosys_mods) self._histosys_histoset = [[[ builder_data[m][s]['data']['lo_data'], builder_data[m][s]['data']['nom_data'], builder_data[m][s]['data']['hi_data'], ] for s in pdfconfig.samples] for m in keys] self._histosys_mask = [[[builder_data[m][s]['data']['mask']] for s in pdfconfig.samples] for m in keys] if histosys_mods: self.interpolator = getattr(interpolators, self.interpcode)( self._histosys_histoset) self._precompute() events.subscribe('tensorlib_changed')(self._precompute)
def test_paramviewer_simple_batched(backend): pars = pyhf.tensorlib.astensor([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]) parshape = pyhf.tensorlib.shape(pars) view = ParamViewer( parshape, { 'hello': { 'slice': slice(0, 2) }, 'world': { 'slice': slice(3, 4) } }, ['world', 'hello'], ) par_slice = view.get(pars) assert isinstance(view.index_selection, list) assert all([len(x) == 3 for x in view.index_selection]) # first dimension is batch dim assert pyhf.tensorlib.shape(par_slice) == (3, 3) assert pyhf.tensorlib.tolist(par_slice[slice(1, 3)]) == [[1, 5, 9], [2, 6, 10]] assert pyhf.tensorlib.tolist(par_slice[slice(0, 1)]) == [[4, 8, 12]] assert pyhf.tensorlib.tolist(par_slice) == [[4, 8, 12], [1, 5, 9], [2, 6, 10]]
class normfactor_combined: name = 'normfactor' op_code = 'multiplication' def __init__(self, modifiers, pdfconfig, builder_data, batch_size=None): self.batch_size = batch_size keys = [f'{mtype}/{m}' for m, mtype in modifiers] normfactor_mods = [m for m, _ in modifiers] parfield_shape = ((self.batch_size, pdfconfig.npars) if self.batch_size else (pdfconfig.npars, )) self.param_viewer = ParamViewer(parfield_shape, pdfconfig.par_map, normfactor_mods) self._normfactor_mask = [[[builder_data[m][s]['data']['mask']] for s in pdfconfig.samples] for m in keys] self._precompute() events.subscribe('tensorlib_changed')(self._precompute) def _precompute(self): tensorlib, _ = get_backend() if not self.param_viewer.index_selection: return self.normfactor_mask = tensorlib.tile( tensorlib.astensor(self._normfactor_mask), (1, 1, self.batch_size or 1, 1)) self.normfactor_mask_bool = tensorlib.astensor(self.normfactor_mask, dtype="bool") self.normfactor_default = tensorlib.ones(self.normfactor_mask.shape) def apply(self, pars): """ Returns: modification tensor: Shape (n_modifiers, n_global_samples, n_alphas, n_global_bin) """ if not self.param_viewer.index_selection: return tensorlib, _ = get_backend() if self.batch_size is None: normfactors = self.param_viewer.get(pars) results_normfactor = tensorlib.einsum('msab,m->msab', self.normfactor_mask, normfactors) else: normfactors = self.param_viewer.get(pars) results_normfactor = tensorlib.einsum('msab,ma->msab', self.normfactor_mask, normfactors) results_normfactor = tensorlib.where(self.normfactor_mask_bool, results_normfactor, self.normfactor_default) return results_normfactor
def __init__(self, config, batch_size): self.batch_size = batch_size self.config = config self.constraints_gaussian = gaussian_constraint_combined( config, batch_size=self.batch_size) self.constraints_poisson = poisson_constraint_combined( config, batch_size=self.batch_size) self.viewer_aux = ParamViewer( (self.batch_size or 1, self.config.npars), self.config.par_map, self.config.auxdata_order, ) assert self.constraints_gaussian.batch_size == self.batch_size assert self.constraints_poisson.batch_size == self.batch_size indices = [] if self.constraints_gaussian.has_pdf(): indices.append(self.constraints_gaussian._normal_data) if self.constraints_poisson.has_pdf(): indices.append(self.constraints_poisson._poisson_data) if self.has_pdf(): self.constraints_tv = _TensorViewer(indices, self.batch_size)
def __init__(self, modifiers, pdfconfig, builder_data, batch_size=None): default_backend = pyhf.default_backend self.batch_size = batch_size keys = [f'{mtype}/{m}' for m, mtype in modifiers] self._shapesys_mods = [m for m, _ in modifiers] parfield_shape = (self.batch_size or 1, pdfconfig.npars) self.param_viewer = ParamViewer(parfield_shape, pdfconfig.par_map, self._shapesys_mods) self._shapesys_mask = [[[builder_data[m][s]['data']['mask']] for s in pdfconfig.samples] for m in keys] self.__shapesys_info = default_backend.astensor([[[ builder_data[m][s]['data']['mask'], builder_data[m][s]['data']['nom_data'], builder_data[m][s]['data']['uncrt'], ] for s in pdfconfig.samples] for m in keys]) global_concatenated_bin_indices = [[[ j for c in pdfconfig.channels for j in range(pdfconfig.channel_nbins[c]) ]]] self._access_field = default_backend.tile( global_concatenated_bin_indices, (len(self._shapesys_mods), self.batch_size or 1, 1), ) # access field is shape (sys, batch, globalbin) # reindex it based on current masking self._reindex_access_field(pdfconfig) self._precompute() events.subscribe('tensorlib_changed')(self._precompute)
def __init__(self, modifiers, pdfconfig, builder_data, batch_size=None): self.batch_size = batch_size keys = [f'{mtype}/{m}' for m, mtype in modifiers] lumi_mods = [m for m, _ in modifiers] parfield_shape = ((self.batch_size, pdfconfig.npars) if self.batch_size else (pdfconfig.npars, )) self.param_viewer = ParamViewer(parfield_shape, pdfconfig.par_map, lumi_mods) self._lumi_mask = [[[builder_data[m][s]['data']['mask']] for s in pdfconfig.samples] for m in keys] self._precompute() events.subscribe('tensorlib_changed')(self._precompute)
def __init__(self, pdfconfig, batch_size=None): default_backend = pyhf.default_backend self.batch_size = batch_size # iterate over all constraints order doesn't matter.... self.par_indices = list(range(pdfconfig.npars)) self.data_indices = list(range(len(pdfconfig.auxdata))) self.parsets = [ pdfconfig.param_set(cname) for cname in pdfconfig.auxdata_order ] pars_constrained_by_poisson = [ constrained_parameter for constrained_parameter in pdfconfig.auxdata_order if pdfconfig.param_set(constrained_parameter).pdf_type == 'poisson' ] parfield_shape = (self.batch_size or 1, pdfconfig.npars) self.param_viewer = ParamViewer(parfield_shape, pdfconfig.par_map, pars_constrained_by_poisson) start_index = 0 poisson_constraint_data = [] poisson_constraint_rate_factors = [] for parset in self.parsets: end_index = start_index + parset.n_parameters thisauxdata = self.data_indices[start_index:end_index] start_index = end_index if not parset.pdf_type == 'poisson': continue poisson_constraint_data.append(thisauxdata) poisson_constraint_rate_factors.append(parset.factors) self._poisson_data = None self._access_field = None self._batched_factors = None if self.param_viewer.index_selection: self._poisson_data = default_backend.astensor( default_backend.concatenate(poisson_constraint_data), dtype='int') _poisson_rate_fac = default_backend.astensor( default_backend.concatenate(poisson_constraint_rate_factors), dtype='float', ) factors = default_backend.reshape(_poisson_rate_fac, (1, -1)) self._batched_factors = default_backend.tile( factors, (self.batch_size or 1, 1)) access_field = default_backend.concatenate( self.param_viewer.index_selection, axis=1) self._access_field = access_field self._precompute() events.subscribe('tensorlib_changed')(self._precompute)
def test_paramviewer_order(get_json_from_tarfile): sbottom_archive = data_path("pyhf-ins1748602-probability-models.tar.gz") lhood = get_json_from_tarfile(sbottom_archive, "RegionA/BkgOnly.json") patch = get_json_from_tarfile(sbottom_archive, "RegionA/patch.sbottom_1300_205_60.json") workspace = pyhf.workspace.Workspace(lhood) model = workspace.model(patches=[patch]) pv = ParamViewer((model.config.npars, ), model.config.par_map, []) assert list(pv.allpar_viewer.names) == model.config.par_order
def test_paramviewer_order(sbottom_likelihoods_download, get_json_from_tarfile): lhood = get_json_from_tarfile(sbottom_likelihoods_download, "RegionA/BkgOnly.json") patch = get_json_from_tarfile(sbottom_likelihoods_download, "RegionA/patch.sbottom_1300_205_60.json") workspace = pyhf.workspace.Workspace(lhood) model = workspace.model(patches=[patch]) pv = ParamViewer((model.config.npars, ), model.config.par_map, []) assert list(pv.allpar_viewer.names) == model.config.par_order
def test_paramviewer_simple_nonbatched(backend): pars = pyhf.tensorlib.astensor([1, 2, 3, 4, 5, 6, 7]) parshape = pyhf.tensorlib.shape(pars) view = ParamViewer( parshape, { 'hello': { 'slice': slice(0, 2) }, 'world': { 'slice': slice(5, 7) } }, ['world', 'hello'], ) par_slice = view.get(pars) assert pyhf.tensorlib.tolist(par_slice[slice(2, 4)]) == [1, 2] assert pyhf.tensorlib.tolist(par_slice[slice(0, 2)]) == [6, 7] assert pyhf.tensorlib.tolist(par_slice) == [6, 7, 1, 2]
class histosys_combined: name = 'histosys' op_code = 'addition' def __init__(self, modifiers, pdfconfig, builder_data, interpcode='code0', batch_size=None): self.batch_size = batch_size self.interpcode = interpcode assert self.interpcode in ['code0', 'code2', 'code4p'] keys = [f'{mtype}/{m}' for m, mtype in modifiers] histosys_mods = [m for m, _ in modifiers] parfield_shape = ((self.batch_size, pdfconfig.npars) if self.batch_size else (pdfconfig.npars, )) self.param_viewer = ParamViewer(parfield_shape, pdfconfig.par_map, histosys_mods) self._histosys_histoset = [[[ builder_data[m][s]['data']['lo_data'], builder_data[m][s]['data']['nom_data'], builder_data[m][s]['data']['hi_data'], ] for s in pdfconfig.samples] for m in keys] self._histosys_mask = [[[builder_data[m][s]['data']['mask']] for s in pdfconfig.samples] for m in keys] if histosys_mods: self.interpolator = getattr(interpolators, self.interpcode)( self._histosys_histoset) self._precompute() events.subscribe('tensorlib_changed')(self._precompute) def _precompute(self): if not self.param_viewer.index_selection: return tensorlib, _ = get_backend() self.histosys_mask = tensorlib.astensor(self._histosys_mask, dtype="bool") self.histosys_default = tensorlib.zeros(self.histosys_mask.shape) if self.batch_size is None: self.indices = tensorlib.reshape( self.param_viewer.indices_concatenated, (-1, 1)) def apply(self, pars): """ Returns: modification tensor: Shape (n_modifiers, n_global_samples, n_alphas, n_global_bin) """ if not self.param_viewer.index_selection: return tensorlib, _ = get_backend() if self.batch_size is None: histosys_alphaset = self.param_viewer.get(pars, self.indices) else: histosys_alphaset = self.param_viewer.get(pars) results_histo = self.interpolator(histosys_alphaset) # either rely on numerical no-op or force with line below results_histo = tensorlib.where(self.histosys_mask, results_histo, self.histosys_default) return results_histo
def __init__(self, pdfconfig, batch_size=None): default_backend = pyhf.default_backend self.batch_size = batch_size # iterate over all constraints order doesn't matter.... self.data_indices = list(range(len(pdfconfig.auxdata))) self.parsets = [ pdfconfig.param_set(cname) for cname in pdfconfig.auxdata_order ] pars_constrained_by_normal = [ constrained_parameter for constrained_parameter in pdfconfig.auxdata_order if pdfconfig.param_set(constrained_parameter).pdf_type == 'normal' ] parfield_shape = (self.batch_size or 1, pdfconfig.npars) self.param_viewer = ParamViewer(parfield_shape, pdfconfig.par_map, pars_constrained_by_normal) start_index = 0 normal_constraint_data = [] normal_constraint_sigmas = [] # loop over parameters (in auxdata order) and collect # means / sigmas of constraint term as well as data # skip parsets that are not constrained by onrmal for parset in self.parsets: end_index = start_index + parset.n_parameters thisauxdata = self.data_indices[start_index:end_index] start_index = end_index if not parset.pdf_type == 'normal': continue normal_constraint_data.append(thisauxdata) # many constraints are defined on a unit gaussian # but we reserved the possibility that a paramset # can define non-standard uncertainties. This is used # by the paramset associated to staterror modifiers. # Such parsets define a 'sigmas' attribute try: normal_constraint_sigmas.append(parset.sigmas) except AttributeError: normal_constraint_sigmas.append([1.0] * len(thisauxdata)) self._normal_data = None self._sigmas = None self._access_field = None # if this constraint terms is at all used (non-zrto idx selection # start preparing constant tensors if self.param_viewer.index_selection: self._normal_data = default_backend.astensor( default_backend.concatenate(normal_constraint_data), dtype='int') _normal_sigmas = default_backend.concatenate( normal_constraint_sigmas) if self.batch_size: sigmas = default_backend.reshape(_normal_sigmas, (1, -1)) self._sigmas = default_backend.tile(sigmas, (self.batch_size, 1)) else: self._sigmas = _normal_sigmas access_field = default_backend.concatenate( self.param_viewer.index_selection, axis=1) self._access_field = access_field self._precompute() events.subscribe('tensorlib_changed')(self._precompute)