def __init__(self,
modifiers,
pdfconfig,
builder_data,
interpcode='code0',
batch_size=None):
self.batch_size = batch_size
self.interpcode = interpcode
assert self.interpcode in ['code0', 'code2', 'code4p']
keys = [f'{mtype}/{m}' for m, mtype in modifiers]
histosys_mods = [m for m, _ in modifiers]
parfield_shape = ((self.batch_size,
pdfconfig.npars) if self.batch_size else
(pdfconfig.npars, ))
self.param_viewer = ParamViewer(parfield_shape, pdfconfig.par_map,
histosys_mods)
self._histosys_histoset = [[[
builder_data[m][s]['data']['lo_data'],
builder_data[m][s]['data']['nom_data'],
builder_data[m][s]['data']['hi_data'],
] for s in pdfconfig.samples] for m in keys]
self._histosys_mask = [[[builder_data[m][s]['data']['mask']]
for s in pdfconfig.samples] for m in keys]
if histosys_mods:
self.interpolator = getattr(interpolators, self.interpcode)(
self._histosys_histoset)
self._precompute()
events.subscribe('tensorlib_changed')(self._precompute)
def test_paramviewer_simple_batched(backend):
pars = pyhf.tensorlib.astensor([[1, 2, 3, 4], [5, 6, 7, 8],
[9, 10, 11, 12]])
parshape = pyhf.tensorlib.shape(pars)
view = ParamViewer(
parshape,
{
'hello': {
'slice': slice(0, 2)
},
'world': {
'slice': slice(3, 4)
}
},
['world', 'hello'],
)
par_slice = view.get(pars)
assert isinstance(view.index_selection, list)
assert all([len(x) == 3
for x in view.index_selection]) # first dimension is batch dim
assert pyhf.tensorlib.shape(par_slice) == (3, 3)
assert pyhf.tensorlib.tolist(par_slice[slice(1, 3)]) == [[1, 5, 9],
[2, 6, 10]]
assert pyhf.tensorlib.tolist(par_slice[slice(0, 1)]) == [[4, 8, 12]]
assert pyhf.tensorlib.tolist(par_slice) == [[4, 8, 12], [1, 5, 9],
[2, 6, 10]]
class normfactor_combined:
name = 'normfactor'
op_code = 'multiplication'
def __init__(self, modifiers, pdfconfig, builder_data, batch_size=None):
self.batch_size = batch_size
keys = [f'{mtype}/{m}' for m, mtype in modifiers]
normfactor_mods = [m for m, _ in modifiers]
parfield_shape = ((self.batch_size,
pdfconfig.npars) if self.batch_size else
(pdfconfig.npars, ))
self.param_viewer = ParamViewer(parfield_shape, pdfconfig.par_map,
normfactor_mods)
self._normfactor_mask = [[[builder_data[m][s]['data']['mask']]
for s in pdfconfig.samples] for m in keys]
self._precompute()
events.subscribe('tensorlib_changed')(self._precompute)
def _precompute(self):
tensorlib, _ = get_backend()
if not self.param_viewer.index_selection:
return
self.normfactor_mask = tensorlib.tile(
tensorlib.astensor(self._normfactor_mask),
(1, 1, self.batch_size or 1, 1))
self.normfactor_mask_bool = tensorlib.astensor(self.normfactor_mask,
dtype="bool")
self.normfactor_default = tensorlib.ones(self.normfactor_mask.shape)
def apply(self, pars):
"""
Returns:
modification tensor: Shape (n_modifiers, n_global_samples, n_alphas, n_global_bin)
"""
if not self.param_viewer.index_selection:
return
tensorlib, _ = get_backend()
if self.batch_size is None:
normfactors = self.param_viewer.get(pars)
results_normfactor = tensorlib.einsum('msab,m->msab',
self.normfactor_mask,
normfactors)
else:
normfactors = self.param_viewer.get(pars)
results_normfactor = tensorlib.einsum('msab,ma->msab',
self.normfactor_mask,
normfactors)
results_normfactor = tensorlib.where(self.normfactor_mask_bool,
results_normfactor,
self.normfactor_default)
return results_normfactor
def __init__(self, config, batch_size):
self.batch_size = batch_size
self.config = config
self.constraints_gaussian = gaussian_constraint_combined(
config, batch_size=self.batch_size)
self.constraints_poisson = poisson_constraint_combined(
config, batch_size=self.batch_size)
self.viewer_aux = ParamViewer(
(self.batch_size or 1, self.config.npars),
self.config.par_map,
self.config.auxdata_order,
)
assert self.constraints_gaussian.batch_size == self.batch_size
assert self.constraints_poisson.batch_size == self.batch_size
indices = []
if self.constraints_gaussian.has_pdf():
indices.append(self.constraints_gaussian._normal_data)
if self.constraints_poisson.has_pdf():
indices.append(self.constraints_poisson._poisson_data)
if self.has_pdf():
self.constraints_tv = _TensorViewer(indices, self.batch_size)
def __init__(self, modifiers, pdfconfig, builder_data, batch_size=None):
default_backend = pyhf.default_backend
self.batch_size = batch_size
keys = [f'{mtype}/{m}' for m, mtype in modifiers]
self._shapesys_mods = [m for m, _ in modifiers]
parfield_shape = (self.batch_size or 1, pdfconfig.npars)
self.param_viewer = ParamViewer(parfield_shape, pdfconfig.par_map,
self._shapesys_mods)
self._shapesys_mask = [[[builder_data[m][s]['data']['mask']]
for s in pdfconfig.samples] for m in keys]
self.__shapesys_info = default_backend.astensor([[[
builder_data[m][s]['data']['mask'],
builder_data[m][s]['data']['nom_data'],
builder_data[m][s]['data']['uncrt'],
] for s in pdfconfig.samples] for m in keys])
global_concatenated_bin_indices = [[[
j for c in pdfconfig.channels
for j in range(pdfconfig.channel_nbins[c])
]]]
self._access_field = default_backend.tile(
global_concatenated_bin_indices,
(len(self._shapesys_mods), self.batch_size or 1, 1),
)
# access field is shape (sys, batch, globalbin)
# reindex it based on current masking
self._reindex_access_field(pdfconfig)
self._precompute()
events.subscribe('tensorlib_changed')(self._precompute)
def __init__(self, modifiers, pdfconfig, builder_data, batch_size=None):
self.batch_size = batch_size
keys = [f'{mtype}/{m}' for m, mtype in modifiers]
lumi_mods = [m for m, _ in modifiers]
parfield_shape = ((self.batch_size,
pdfconfig.npars) if self.batch_size else
(pdfconfig.npars, ))
self.param_viewer = ParamViewer(parfield_shape, pdfconfig.par_map,
lumi_mods)
self._lumi_mask = [[[builder_data[m][s]['data']['mask']]
for s in pdfconfig.samples] for m in keys]
self._precompute()
events.subscribe('tensorlib_changed')(self._precompute)
def __init__(self, pdfconfig, batch_size=None):
default_backend = pyhf.default_backend
self.batch_size = batch_size
# iterate over all constraints order doesn't matter....
self.par_indices = list(range(pdfconfig.npars))
self.data_indices = list(range(len(pdfconfig.auxdata)))
self.parsets = [
pdfconfig.param_set(cname) for cname in pdfconfig.auxdata_order
]
pars_constrained_by_poisson = [
constrained_parameter
for constrained_parameter in pdfconfig.auxdata_order
if pdfconfig.param_set(constrained_parameter).pdf_type == 'poisson'
]
parfield_shape = (self.batch_size or 1, pdfconfig.npars)
self.param_viewer = ParamViewer(parfield_shape, pdfconfig.par_map,
pars_constrained_by_poisson)
start_index = 0
poisson_constraint_data = []
poisson_constraint_rate_factors = []
for parset in self.parsets:
end_index = start_index + parset.n_parameters
thisauxdata = self.data_indices[start_index:end_index]
start_index = end_index
if not parset.pdf_type == 'poisson':
continue
poisson_constraint_data.append(thisauxdata)
poisson_constraint_rate_factors.append(parset.factors)
self._poisson_data = None
self._access_field = None
self._batched_factors = None
if self.param_viewer.index_selection:
self._poisson_data = default_backend.astensor(
default_backend.concatenate(poisson_constraint_data),
dtype='int')
_poisson_rate_fac = default_backend.astensor(
default_backend.concatenate(poisson_constraint_rate_factors),
dtype='float',
)
factors = default_backend.reshape(_poisson_rate_fac, (1, -1))
self._batched_factors = default_backend.tile(
factors, (self.batch_size or 1, 1))
access_field = default_backend.concatenate(
self.param_viewer.index_selection, axis=1)
self._access_field = access_field
self._precompute()
events.subscribe('tensorlib_changed')(self._precompute)
def test_paramviewer_order(get_json_from_tarfile):
sbottom_archive = data_path("pyhf-ins1748602-probability-models.tar.gz")
lhood = get_json_from_tarfile(sbottom_archive, "RegionA/BkgOnly.json")
patch = get_json_from_tarfile(sbottom_archive,
"RegionA/patch.sbottom_1300_205_60.json")
workspace = pyhf.workspace.Workspace(lhood)
model = workspace.model(patches=[patch])
pv = ParamViewer((model.config.npars, ), model.config.par_map, [])
assert list(pv.allpar_viewer.names) == model.config.par_order
def test_paramviewer_order(sbottom_likelihoods_download,
get_json_from_tarfile):
lhood = get_json_from_tarfile(sbottom_likelihoods_download,
"RegionA/BkgOnly.json")
patch = get_json_from_tarfile(sbottom_likelihoods_download,
"RegionA/patch.sbottom_1300_205_60.json")
workspace = pyhf.workspace.Workspace(lhood)
model = workspace.model(patches=[patch])
pv = ParamViewer((model.config.npars, ), model.config.par_map, [])
assert list(pv.allpar_viewer.names) == model.config.par_order
def test_paramviewer_simple_nonbatched(backend):
pars = pyhf.tensorlib.astensor([1, 2, 3, 4, 5, 6, 7])
parshape = pyhf.tensorlib.shape(pars)
view = ParamViewer(
parshape,
{
'hello': {
'slice': slice(0, 2)
},
'world': {
'slice': slice(5, 7)
}
},
['world', 'hello'],
)
par_slice = view.get(pars)
assert pyhf.tensorlib.tolist(par_slice[slice(2, 4)]) == [1, 2]
assert pyhf.tensorlib.tolist(par_slice[slice(0, 2)]) == [6, 7]
assert pyhf.tensorlib.tolist(par_slice) == [6, 7, 1, 2]
class histosys_combined:
name = 'histosys'
op_code = 'addition'
def __init__(self,
modifiers,
pdfconfig,
builder_data,
interpcode='code0',
batch_size=None):
self.batch_size = batch_size
self.interpcode = interpcode
assert self.interpcode in ['code0', 'code2', 'code4p']
keys = [f'{mtype}/{m}' for m, mtype in modifiers]
histosys_mods = [m for m, _ in modifiers]
parfield_shape = ((self.batch_size,
pdfconfig.npars) if self.batch_size else
(pdfconfig.npars, ))
self.param_viewer = ParamViewer(parfield_shape, pdfconfig.par_map,
histosys_mods)
self._histosys_histoset = [[[
builder_data[m][s]['data']['lo_data'],
builder_data[m][s]['data']['nom_data'],
builder_data[m][s]['data']['hi_data'],
] for s in pdfconfig.samples] for m in keys]
self._histosys_mask = [[[builder_data[m][s]['data']['mask']]
for s in pdfconfig.samples] for m in keys]
if histosys_mods:
self.interpolator = getattr(interpolators, self.interpcode)(
self._histosys_histoset)
self._precompute()
events.subscribe('tensorlib_changed')(self._precompute)
def _precompute(self):
if not self.param_viewer.index_selection:
return
tensorlib, _ = get_backend()
self.histosys_mask = tensorlib.astensor(self._histosys_mask,
dtype="bool")
self.histosys_default = tensorlib.zeros(self.histosys_mask.shape)
if self.batch_size is None:
self.indices = tensorlib.reshape(
self.param_viewer.indices_concatenated, (-1, 1))
def apply(self, pars):
"""
Returns:
modification tensor: Shape (n_modifiers, n_global_samples, n_alphas, n_global_bin)
"""
if not self.param_viewer.index_selection:
return
tensorlib, _ = get_backend()
if self.batch_size is None:
histosys_alphaset = self.param_viewer.get(pars, self.indices)
else:
histosys_alphaset = self.param_viewer.get(pars)
results_histo = self.interpolator(histosys_alphaset)
# either rely on numerical no-op or force with line below
results_histo = tensorlib.where(self.histosys_mask, results_histo,
self.histosys_default)
return results_histo
def __init__(self, pdfconfig, batch_size=None):
default_backend = pyhf.default_backend
self.batch_size = batch_size
# iterate over all constraints order doesn't matter....
self.data_indices = list(range(len(pdfconfig.auxdata)))
self.parsets = [
pdfconfig.param_set(cname) for cname in pdfconfig.auxdata_order
]
pars_constrained_by_normal = [
constrained_parameter
for constrained_parameter in pdfconfig.auxdata_order
if pdfconfig.param_set(constrained_parameter).pdf_type == 'normal'
]
parfield_shape = (self.batch_size or 1, pdfconfig.npars)
self.param_viewer = ParamViewer(parfield_shape, pdfconfig.par_map,
pars_constrained_by_normal)
start_index = 0
normal_constraint_data = []
normal_constraint_sigmas = []
# loop over parameters (in auxdata order) and collect
# means / sigmas of constraint term as well as data
# skip parsets that are not constrained by onrmal
for parset in self.parsets:
end_index = start_index + parset.n_parameters
thisauxdata = self.data_indices[start_index:end_index]
start_index = end_index
if not parset.pdf_type == 'normal':
continue
normal_constraint_data.append(thisauxdata)
# many constraints are defined on a unit gaussian
# but we reserved the possibility that a paramset
# can define non-standard uncertainties. This is used
# by the paramset associated to staterror modifiers.
# Such parsets define a 'sigmas' attribute
try:
normal_constraint_sigmas.append(parset.sigmas)
except AttributeError:
normal_constraint_sigmas.append([1.0] * len(thisauxdata))
self._normal_data = None
self._sigmas = None
self._access_field = None
# if this constraint terms is at all used (non-zrto idx selection
# start preparing constant tensors
if self.param_viewer.index_selection:
self._normal_data = default_backend.astensor(
default_backend.concatenate(normal_constraint_data),
dtype='int')
_normal_sigmas = default_backend.concatenate(
normal_constraint_sigmas)
if self.batch_size:
sigmas = default_backend.reshape(_normal_sigmas, (1, -1))
self._sigmas = default_backend.tile(sigmas,
(self.batch_size, 1))
else:
self._sigmas = _normal_sigmas
access_field = default_backend.concatenate(
self.param_viewer.index_selection, axis=1)
self._access_field = access_field
self._precompute()
events.subscribe('tensorlib_changed')(self._precompute)