def test_invalid_modifier_name_resuse(): spec = { 'channels': [{ 'name': 'singlechannel', 'samples': [{ 'name': 'signal', 'data': [5.], 'modifiers': [{ 'name': 'reused_name', 'type': 'normfactor', 'data': None }] }, { 'name': 'background', 'data': [50.], 'modifiers': [{ 'name': 'reused_name', 'type': 'normsys', 'data': { 'lo': 0.9, 'hi': 1.1 } }] }] }] } with pytest.raises(pyhf.exceptions.InvalidNameReuse): pdf = pyhf.Model(spec, poiname='reused_name') pdf = pyhf.Model(spec, poiname='reused_name', qualify_names=True)
def test_invalid_bin_wise_modifier(datadir, patch_file): """ Test that bin-wise modifiers will raise an exception if their data shape differs from their sample's. """ spec = json.load(open(datadir.join("spec.json"))) assert pyhf.Model(spec) patch = JsonPatch.from_string(open(datadir.join(patch_file)).read()) bad_spec = patch.apply(spec) with pytest.raises(pyhf.exceptions.InvalidModifier): pyhf.Model(bad_spec)
def test_parameters_fixed(): spec = { 'channels': [{ 'name': 'channel', 'samples': [ { 'name': 'sample', 'data': [10.0], 'modifiers': [{ 'name': 'unfixed', 'type': 'normfactor', 'data': None }], }, { 'name': 'another_sample', 'data': [5.0], 'modifiers': [{ 'name': 'mypoi', 'type': 'normfactor', 'data': None }], }, ], }], 'parameters': [{ 'name': 'mypoi', 'inits': [1], 'fixed': True }], } pyhf.Model(spec, poi_name='mypoi')
def test_shapefactor_build(): spec = { 'channels': [{ 'name': 'channel', 'samples': [ { 'name': 'sample', 'data': [10.0] * 3, 'modifiers': [ { 'name': 'mu', 'type': 'normfactor', 'data': None }, ], }, { 'name': 'another_sample', 'data': [5.0] * 3, 'modifiers': [{ 'name': 'freeshape', 'type': 'shapefactor', 'data': None }], }, ], }], } model = pyhf.Model(spec) assert model
def test_optim_correlations(backend, source, spec, mu): pdf = pyhf.Model(spec) data = source['bindata']['data'] + pdf.config.auxdata init_pars = pdf.config.suggested_init() par_bounds = pdf.config.suggested_bounds() optim = pyhf.optimizer result = optim.minimize(pyhf.infer.mle.twice_nll, data, pdf, init_pars, par_bounds) assert pyhf.tensorlib.tolist(result) result, correlations = optim.minimize( pyhf.infer.mle.twice_nll, data, pdf, init_pars, par_bounds, [(pdf.config.poi_index, mu)], return_correlations=True, ) assert result.shape == (2, ) assert correlations.shape == (2, 2) assert pyhf.tensorlib.tolist(result) assert pyhf.tensorlib.tolist(correlations) assert np.allclose([[0.0, 0.0], [0.0, 1.0]], pyhf.tensorlib.tolist(correlations))
def test_validation(setup): source = setup['source'] pdf = pyhf.Model(setup['spec'], modifier_settings={'normsys': { 'interpcode': 'code1' }}) if 'channels' in source: data = [] for c in pdf.config.channels: data += source['channels'][c]['bindata']['data'] data = data + pdf.config.auxdata else: data = source['bindata']['data'] + pdf.config.auxdata if 'auxdata' in setup['expected']['config']: assert len( pdf.config.auxdata) == setup['expected']['config']['auxdata'] assert len(pdf.config.suggested_init() ) == setup['expected']['config']['init_pars'] assert (len(pdf.config.suggested_bounds()) == setup['expected']['config'] ['par_bounds']) validate_hypotest( pdf, data, setup['mu'], setup['expected']['result'], test_stat=setup['test_stat'], tolerance=setup['tolerance'], calctype=setup['calctype'], )
def test_optim_with_value(backend, source, spec, mu): pdf = pyhf.Model(spec) data = source['bindata']['data'] + pdf.config.auxdata init_pars = pdf.config.suggested_init() par_bounds = pdf.config.suggested_bounds() optim = pyhf.optimizer result = optim.minimize(pyhf.infer.mle.twice_nll, data, pdf, init_pars, par_bounds) assert pyhf.tensorlib.tolist(result) result, fitted_val = optim.minimize( pyhf.infer.mle.twice_nll, data, pdf, init_pars, par_bounds, fixed_vals=[(pdf.config.poi_index, mu)], return_fitted_val=True, ) assert pyhf.tensorlib.tolist(result) assert pyhf.tensorlib.shape(fitted_val) == () assert pytest.approx(17.52954975, rel=1e-5) == pyhf.tensorlib.tolist(fitted_val)
def test_model_integration_fixed_parameters_shapesys(): spec = { 'channels': [ { 'name': 'channel', 'samples': [ { 'name': 'sample', 'data': [10.0] * 3, 'modifiers': [ {'name': 'unfixed', 'type': 'normfactor', 'data': None}, {'name': 'uncorr', 'type': 'shapesys', 'data': [1.5] * 3}, ], }, { 'name': 'another_sample', 'data': [5.0] * 3, 'modifiers': [ {'name': 'mypoi', 'type': 'normfactor', 'data': None} ], }, ], } ], 'parameters': [{'name': 'uncorr', 'inits': [1.0, 2.0, 3.0], 'fixed': True}], } model = pyhf.Model(spec, poi_name='mypoi') assert len(model.config.suggested_fixed()) == 5 assert model.config.suggested_fixed() == [False, True, True, True, False] assert model.config.poi_index == 4
def test_override_paramsets_incorrect_num_parameters(): source = json.load(open('validation/data/2bin_histosys_example2.json')) spec = { 'channels': [ { 'name': 'singlechannel', 'samples': [ { 'name': 'signal', 'data': source['bindata']['sig'], 'modifiers': [ {'name': 'mu', 'type': 'normfactor', 'data': None} ], }, { 'name': 'background', 'data': source['bindata']['bkg'], 'modifiers': [ {'name': 'bkg_norm', 'type': 'shapesys', 'data': [10, 10]} ], }, ], } ], 'parameters': [{'name': 'bkg_norm', 'inits': [99, 99], 'bounds': [[95, 95]]}], } with pytest.raises(pyhf.exceptions.InvalidModel): pyhf.Model(spec)
def test_validation(setup_and_tolerance): setup, tolerance = setup_and_tolerance source = setup['source'] pdf = pyhf.Model(setup['spec']) if 'channels' in source: data = [] for c in pdf.config.channels: data += source['channels'][c]['bindata']['data'] data = data + pdf.config.auxdata else: data = source['bindata']['data'] + pdf.config.auxdata if 'auxdata' in setup['expected']['config']: assert len( pdf.config.auxdata) == setup['expected']['config']['auxdata'] assert len(pdf.config.suggested_init() ) == setup['expected']['config']['init_pars'] assert (len(pdf.config.suggested_bounds()) == setup['expected']['config'] ['par_bounds']) validate_hypotest(pdf, data, setup['mu'], setup['expected']['result'], tolerance=tolerance)
def test_import_shapesys(): parsed_xml = pyhf.readxml.parse( 'validation/xmlimport_input3/config/examples/example_ShapeSys.xml', 'validation/xmlimport_input3', ) # build the spec, strictly checks properties included spec = { 'channels': parsed_xml['channels'], 'parameters': parsed_xml['measurements'][0]['config']['parameters'], } pdf = pyhf.Model(spec, poiname='SigXsecOverSM') data = [ binvalue for k in pdf.spec['channels'] for binvalue in next(obs for obs in parsed_xml['observations'] if obs['name'] == k['name'])['data'] ] + pdf.config.auxdata channels = {channel['name']: channel for channel in pdf.spec['channels']} samples = { channel['name']: [sample['name'] for sample in channel['samples']] for channel in pdf.spec['channels'] } assert channels['channel1']['samples'][1]['modifiers'][0]['type'] == 'lumi' assert channels['channel1']['samples'][1]['modifiers'][1][ 'type'] == 'shapesys' # NB: assert that relative uncertainty is converted to absolute uncertainty for shapesys assert channels['channel1']['samples'][1]['data'] == pytest.approx( [100.0, 1.0e-4]) assert channels['channel1']['samples'][1]['modifiers'][1][ 'data'] == pytest.approx([10.0, 1.5e-5])
def test_override_paramset_defaults(): source = json.load(open('validation/data/2bin_histosys_example2.json')) spec = { 'channels': [ { 'name': 'singlechannel', 'samples': [ { 'name': 'signal', 'data': source['bindata']['sig'], 'modifiers': [ {'name': 'mu', 'type': 'normfactor', 'data': None} ], }, { 'name': 'background', 'data': source['bindata']['bkg'], 'modifiers': [ {'name': 'bkg_norm', 'type': 'shapesys', 'data': [10, 10]} ], }, ], } ], 'parameters': [ {'name': 'bkg_norm', 'inits': [99, 99], 'bounds': [[95, 95], [95, 95]]} ], } pdf = pyhf.Model(spec) assert pdf.config.param_set('bkg_norm').suggested_bounds == [[95, 95], [95, 95]] assert pdf.config.param_set('bkg_norm').suggested_init == [99, 99]
def test_parameters_incorrect_format(): spec = { 'channels': [{ 'name': 'channel', 'samples': [ { 'name': 'sample', 'data': [10.0], 'modifiers': [] }, { 'name': 'another_sample', 'data': [5.0], 'modifiers': [{ 'name': 'mypoi', 'type': 'normfactor', 'data': None }], }, ], }], 'parameters': { 'a': 'fake', 'object': 2 }, } with pytest.raises(pyhf.exceptions.InvalidSpecification): pyhf.Model(spec, poiname='mypoi')
def test_import_histosys(): parsed_xml = pyhf.readxml.parse( 'validation/xmlimport_input2/config/example.xml', 'validation/xmlimport_input2') # build the spec, strictly checks properties included spec = { 'channels': parsed_xml['channels'], 'parameters': parsed_xml['measurements'][0]['config']['parameters'], } pdf = pyhf.Model(spec, poiname='SigXsecOverSM') data = [ binvalue for k in pdf.spec['channels'] for binvalue in next(obs for obs in parsed_xml['observations'] if obs['name'] == k['name'])['data'] ] + pdf.config.auxdata channels = {channel['name']: channel for channel in pdf.spec['channels']} samples = { channel['name']: [sample['name'] for sample in channel['samples']] for channel in pdf.spec['channels'] } assert channels['channel2']['samples'][0]['modifiers'][0]['type'] == 'lumi' assert channels['channel2']['samples'][0]['modifiers'][1][ 'type'] == 'histosys'
def test_poiless_model(backend): spec = { 'channels': [{ 'name': 'channel', 'samples': [ { 'name': 'goodsample', 'data': [10.0], 'modifiers': [{ 'type': 'normsys', 'name': 'shape', 'data': { "hi": 0.5, "lo": 1.5 }, }], }, ], }] } model = pyhf.Model(spec, poi_name=None) data = [12] + model.config.auxdata pyhf.infer.mle.fit(data, model) with pytest.raises(pyhf.exceptions.UnspecifiedPOI): pyhf.infer.mle.fixed_poi_fit(1.0, data, model) with pytest.raises(pyhf.exceptions.UnspecifiedPOI): pyhf.infer.hypotest(1.0, data, model)
def test_parameters_normfactor_bad_attribute(bad_parameter): spec = { 'channels': [{ 'name': 'channel', 'samples': [ { 'name': 'sample', 'data': [10.0], 'modifiers': [] }, { 'name': 'another_sample', 'data': [5.0], 'modifiers': [{ 'name': 'mypoi', 'type': 'normfactor', 'data': None }], }, ], }], 'parameters': [bad_parameter], } with pytest.raises(pyhf.exceptions.InvalidModel): pyhf.Model(spec, poiname='mypoi')
def test_one_sample_missing_modifiers(): spec = { 'channels': [{ 'name': 'channel', 'samples': [ { 'name': 'sample', 'data': [10.0], 'modifiers': [] }, { 'name': 'another_sample', 'data': [5.0], 'modifiers': [{ 'name': 'mypoi', 'type': 'normfactor', 'data': None }], }, ], }] } pyhf.Model(spec, poiname='mypoi')
def test_parameters_all_props(): spec = { 'channels': [{ 'name': 'channel', 'samples': [ { 'name': 'sample', 'data': [10.0], 'modifiers': [] }, { 'name': 'another_sample', 'data': [5.0], 'modifiers': [{ 'name': 'mypoi', 'type': 'normfactor', 'data': None }], }, ], }], 'parameters': [{ 'name': 'mypoi', 'inits': [1], 'bounds': [[0, 1]] }], } pyhf.Model(spec, poiname='mypoi')
def test_parameters_definition(): spec = { 'channels': [{ 'name': 'channel', 'samples': [ { 'name': 'sample', 'data': [10.0], 'modifiers': [] }, { 'name': 'another_sample', 'data': [5.0], 'modifiers': [{ 'name': 'mypoi', 'type': 'normfactor', 'data': None }], }, ], }], 'parameters': [{ 'name': 'mypoi' }], } pyhf.Model(spec, poiname='mypoi')
def test_additional_properties(): spec = { 'channels': [{ 'name': 'channel', 'samples': [ { 'name': 'sample', 'data': [10.0], 'modifiers': [] }, { 'name': 'another_sample', 'data': [5.0], 'modifiers': [{ 'name': 'mypoi', 'type': 'normfactor', 'data': None }], }, ], }], 'fake_additional_property': 2, } with pytest.raises(pyhf.exceptions.InvalidSpecification): pyhf.Model(spec)
def test_optim_uncerts(backend, source, spec, mu): pdf = pyhf.Model(spec) data = source['bindata']['data'] + pdf.config.auxdata init_pars = pdf.config.suggested_init() par_bounds = pdf.config.suggested_bounds() optim = pyhf.optimizer result = optim.minimize(pyhf.infer.mle.twice_nll, data, pdf, init_pars, par_bounds) assert pyhf.tensorlib.tolist(result) result = optim.minimize( pyhf.infer.mle.twice_nll, data, pdf, init_pars, par_bounds, fixed_vals=[(pdf.config.poi_index, mu)], return_uncertainties=True, ) assert result.shape[1] == 2 assert pytest.approx([0.0, 0.26418431]) == pyhf.tensorlib.tolist(result[:, 1])
def test_pdf_integration_normsys(backend): source = json.load(open('validation/data/2bin_histosys_example2.json')) spec = { 'channels': [ { 'name': 'singlechannel', 'samples': [ { 'name': 'signal', 'data': source['bindata']['sig'], 'modifiers': [ {'name': 'mu', 'type': 'normfactor', 'data': None} ], }, { 'name': 'background', 'data': source['bindata']['bkg'], 'modifiers': [ { 'name': 'bkg_norm', 'type': 'normsys', 'data': {'lo': 0.9, 'hi': 1.1}, } ], }, ], } ] } pdf = pyhf.Model(spec) pars = [None, None] pars[pdf.config.par_slice('mu')], pars[pdf.config.par_slice('bkg_norm')] = [ [0.0], [0.0], ] assert np.allclose( pyhf.tensorlib.tolist(pdf.expected_data(pars, include_auxdata=False)), [100, 150], ) pars[pdf.config.par_slice('mu')], pars[pdf.config.par_slice('bkg_norm')] = [ [0.0], [1.0], ] assert np.allclose( pyhf.tensorlib.tolist(pdf.expected_data(pars, include_auxdata=False)), [100 * 1.1, 150 * 1.1], ) pars[pdf.config.par_slice('mu')], pars[pdf.config.par_slice('bkg_norm')] = [ [0.0], [-1.0], ] assert np.allclose( pyhf.tensorlib.tolist(pdf.expected_data(pars, include_auxdata=False)), [100 * 0.9, 150 * 0.9], )
def test_no_poi_test_stats(): spec = { "channels": [{ "name": "channel", "samples": [ { "name": "sample", "data": [10.0], "modifiers": [{ "type": "normsys", "name": "shape", "data": { "hi": 0.5, "lo": 1.5 }, }], }, ], }] } model = pyhf.Model(spec, poi_name=None) test_poi = 1.0 data = [12] + model.config.auxdata init_pars = model.config.suggested_init() par_bounds = model.config.suggested_bounds() fixed_params = model.config.suggested_fixed() with pytest.raises(pyhf.exceptions.UnspecifiedPOI) as excinfo: pyhf.infer.test_statistics.qmu(test_poi, data, model, init_pars, par_bounds, fixed_params) assert ( "No POI is defined. A POI is required for profile likelihood based test statistics." in str(excinfo.value)) with pytest.raises(pyhf.exceptions.UnspecifiedPOI) as excinfo: pyhf.infer.test_statistics.qmu_tilde(test_poi, data, model, init_pars, par_bounds, fixed_params) assert ( "No POI is defined. A POI is required for profile likelihood based test statistics." in str(excinfo.value)) with pytest.raises(pyhf.exceptions.UnspecifiedPOI) as excinfo: pyhf.infer.test_statistics.tmu(test_poi, data, model, init_pars, par_bounds, fixed_params) assert ( "No POI is defined. A POI is required for profile likelihood based test statistics." in str(excinfo.value)) with pytest.raises(pyhf.exceptions.UnspecifiedPOI) as excinfo: pyhf.infer.test_statistics.tmu_tilde(test_poi, data, model, init_pars, par_bounds, fixed_params) assert ( "No POI is defined. A POI is required for profile likelihood based test statistics." in str(excinfo.value))
def test_import_prepHistFactory(): parsed_xml = pyhf.readxml.parse( 'validation/xmlimport_input/config/example.xml', 'validation/xmlimport_input/') # build the spec, strictly checks properties included spec = {'channels': parsed_xml['channels']} pdf = pyhf.Model(spec, poiname='SigXsecOverSM') data = [ binvalue for k in pdf.spec['channels'] for binvalue in parsed_xml['data'][k['name']] ] + pdf.config.auxdata channels = {channel['name'] for channel in pdf.spec['channels']} samples = { channel['name']: [sample['name'] for sample in channel['samples']] for channel in pdf.spec['channels'] } ### ### signal overallsys ### bkg1 overallsys (stat ignored) ### bkg2 stateror (2 bins) ### bkg2 overallsys assert 'channel1' in channels assert 'signal' in samples['channel1'] assert 'background1' in samples['channel1'] assert 'background2' in samples['channel1'] assert pdf.spec['channels'][0]['samples'][2]['modifiers'][0][ 'type'] == 'staterror' assert pdf.spec['channels'][0]['samples'][2]['modifiers'][0]['data'] == [ 0, 10. ] assert pdf.spec['channels'][0]['samples'][1]['modifiers'][0][ 'type'] == 'staterror' assert all( np.isclose( pdf.spec['channels'][0]['samples'][1]['modifiers'][0]['data'], [5.0, 0.0])) assert pdf.expected_actualdata( pdf.config.suggested_init()).tolist() == [120.0, 110.0] assert pdf.config.auxdata_order == [ 'syst1', 'staterror_channel1', 'syst2', 'syst3' ] assert data == [122.0, 112.0, 0.0, 1.0, 1.0, 0.0, 0.0] pars = pdf.config.suggested_init() pars[pdf.config.par_slice('SigXsecOverSM')] = [2.0] assert pdf.expected_data(pars, include_auxdata=False).tolist() == [140, 120]
def test_pdf_integration_shapesys(): source = json.load(open('validation/data/2bin_histosys_example2.json')) spec = { 'channels': [{ 'name': 'singlechannel', 'samples': [{ 'name': 'signal', 'data': source['bindata']['sig'], 'modifiers': [{ 'name': 'mu', 'type': 'normfactor', 'data': None }] }, { 'name': 'background', 'data': source['bindata']['bkg'], 'modifiers': [{ 'name': 'bkg_norm', 'type': 'shapesys', 'data': [10, 10] }] }] }] } pdf = pyhf.Model(spec) pars = [None, None] pars[pdf.config.par_slice('mu')], pars[pdf.config.par_slice( 'bkg_norm')] = [[0.0], [1.0, 1.0]] assert pdf.expected_data(pars, include_auxdata=False).tolist() == [100, 150] pars[pdf.config.par_slice('mu')], pars[pdf.config.par_slice( 'bkg_norm')] = [[0.0], [1.1, 1.0]] assert pdf.expected_data( pars, include_auxdata=False).tolist() == [100 * 1.1, 150] pars[pdf.config.par_slice('mu')], pars[pdf.config.par_slice( 'bkg_norm')] = [[0.0], [1.0, 1.1]] assert pdf.expected_data( pars, include_auxdata=False).tolist() == [100, 150 * 1.1] pars[pdf.config.par_slice('mu')], pars[pdf.config.par_slice( 'bkg_norm')] = [[0.0], [1.1, 0.9]] assert pdf.expected_data( pars, include_auxdata=False).tolist() == [100 * 1.1, 150 * 0.9] pars[pdf.config.par_slice('mu')], pars[pdf.config.par_slice( 'bkg_norm')] = [[0.0], [0.9, 1.1]] assert pdf.expected_data( pars, include_auxdata=False).tolist() == [100 * 0.9, 150 * 1.1]
def test_no_samples(): spec = { 'channels': [ { 'name': 'channel', 'samples': [] }, ] } with pytest.raises(pyhf.exceptions.InvalidSpecification): pyhf.Model(spec)
def test_pdf_integration_staterror(backend): spec = { 'channels': [{ 'name': 'firstchannel', 'samples': [ { 'name': 'mu', 'data': [10.0, 10.0], 'modifiers': [{ 'name': 'mu', 'type': 'normfactor', 'data': None }], }, { 'name': 'bkg1', 'data': [50.0, 70.0], 'modifiers': [{ 'name': 'stat_firstchannel', 'type': 'staterror', 'data': [12.0, 12.0], }], }, { 'name': 'bkg2', 'data': [30.0, 20.0], 'modifiers': [{ 'name': 'stat_firstchannel', 'type': 'staterror', 'data': [5.0, 5.0], }], }, { 'name': 'bkg3', 'data': [20.0, 15.0], 'modifiers': [] }, ], }] } pdf = pyhf.Model(spec) par = pdf.config.par_slice('stat_firstchannel') par_set = pdf.config.param_set('stat_firstchannel') tensorlib, _ = backend uncerts = tensorlib.astensor([[12.0, 12.0], [5.0, 5.0]]) nominal = tensorlib.astensor([[50.0, 70.0], [30.0, 20.0]]) quad = tensorlib.sqrt(tensorlib.sum(tensorlib.power(uncerts, 2), axis=0)) totals = tensorlib.sum(nominal, axis=0) assert pytest.approx(tensorlib.tolist(par_set.sigmas)) == tensorlib.tolist( tensorlib.divide(quad, totals))
def test_sample_missing_data(): spec = { 'channels': [ { 'name': 'channel', 'samples': [{'name': 'sample', 'data': [], 'modifiers': []}], } ] } with pytest.raises(pyhf.exceptions.InvalidSpecification): pyhf.Model(spec)
def test_pdf_integration_staterror(): spec = { 'channels': [ { 'name': 'firstchannel', 'samples': [{ 'name': 'mu', 'data': [10., 10.], 'modifiers': [{ 'name': 'mu', 'type': 'normfactor', 'data': None }] }, { 'name': 'bkg1', 'data': [50.0, 70.0], 'modifiers': [{ 'name': 'stat_firstchannel', 'type': 'staterror', 'data': [12., 12.] }] }, { 'name': 'bkg2', 'data': [30.0, 20.], 'modifiers': [{ 'name': 'stat_firstchannel', 'type': 'staterror', 'data': [5., 5.] }] }, { 'name': 'bkg3', 'data': [20.0, 15.0], 'modifiers': [] }] }, ] } pdf = pyhf.Model(spec) par = pdf.config.par_slice('stat_firstchannel') mod = pdf.config.modifier('stat_firstchannel') assert mod.uncertainties == [[12., 12.], [5., 5.]] assert mod.nominal_counts == [[50., 70.], [30., 20.]] computed = pyhf.tensorlib.tolist(mod.pdf([1.0, 1.0], [1.0, 1.0])) expected = pyhf.tensorlib.tolist( pyhf.tensorlib.normal([1.0, 1.0], mu=[1.0, 1.0], sigma=[13. / 80., 13. / 90.])) for c, e in zip(computed, expected): assert c == e
def test_sample_missing_all_modifiers(): spec = { 'channels': [ { 'name': 'channel', 'samples': [{'name': 'sample', 'data': [10.0], 'modifiers': []}], } ] } with pytest.raises(pyhf.exceptions.InvalidModel): pyhf.Model(spec)