def test_grouped_pha_all_bad_response_bg_warning(elo, ehi, nbins, bkg_id,
caplog, make_data_path,
clean_astro_ui):
"""Check we get the warning messages with background filtering"""
ui.load_pha('check', make_data_path('3c273.pi'))
ui.set_quality('check', 2 * numpy.ones(1024, dtype=numpy.int16), bkg_id=1)
ui.ignore_bad('check', bkg_id=1)
with caplog.at_level(logging.INFO, logger='sherpa'):
ui.notice_id('check', elo, ehi, bkg_id=bkg_id)
# filtering has or hasn't happened
nsrc = ui.get_dep('check', filter=True).size
nback = ui.get_dep('check', filter=True, bkg_id=1).size
if bkg_id is None:
assert nsrc == nbins
assert nback == 0
else:
assert nsrc == 46 # ie no filter
assert nback == 0
# did we get a warning message from the background?
assert len(caplog.records) == 1
name, lvl, msg = caplog.record_tuples[0]
assert name == 'sherpa.astro.data'
assert lvl == logging.INFO
assert msg.startswith('Skipping dataset ')
assert msg.endswith('/3c273_bg.pi: mask excludes all data')
def test_filter_notice_bad_361(make_data_path, caplog):
"""Test out issue 361: notice then ignore bad.
Since the data is grouped, the ignore_bad call is expected to
drop the filter expression, with a warning message.
"""
stats = setup_model(make_data_path)
# We don't care about these warnings, so I could just
# store the number and check that we get an extra one
# below, but use this as a canary to check when the
# system changes.
#
assert len(caplog.records) == 5
ui.notice(0.5, 8.0)
with caplog.at_level(logging.INFO, logger='sherpa'):
ui.ignore_bad()
assert len(caplog.records) == 6
lname, lvl, msg = caplog.record_tuples[5]
assert lname == 'sherpa.astro.data'
assert lvl == logging.WARNING
assert msg == 'filtering grouped data with quality ' + \
'flags, previous filters deleted'
s1 = ui.calc_stat()
assert s1 == pytest.approx(stats['bad'])
def test_filter_notice_bad_361(make_data_path):
"""Test out issue 361: notice then ignore bad.
Since the data is grouped, the ignore_bad call is expected to
drop the filter expression, with a warning message.
"""
logger = logging.getLogger('sherpa')
hdlr = MockLoggingHandler(level='WARNING')
logger.addHandler(hdlr)
stats = setup_model(make_data_path)
ui.notice(0.5, 8.0)
ui.ignore_bad()
s1 = ui.calc_stat()
assert s1 == pytest.approx(stats['bad'])
msgs = hdlr.messages
assert msgs['warning'] == [
'filtering grouped data with quality ' +
'flags, previous filters deleted'
]
for k in ['debug', 'info', 'error', 'critical']:
assert msgs[k] == []
def test_filter_bad_notice_361(make_data_path):
"""Test out issue 361: ignore bad then notice.
This is expected to fail with NumPy version 1.13 and should cause
a DeprecationWarning with earlier versions (I do not know when
the warning was added).
"""
stats = setup_model(make_data_path)
ui.ignore_bad()
ui.notice(0.5, 8.0)
s1 = ui.calc_stat()
assert s1 == pytest.approx(stats['0.5-8.0'])
def test_filter_bad_grouped(make_data_path, clean_astro_ui):
"""Check behavior when the data is grouped.
This is a test of the current behavior, to check that
values still hold. It may be necessary to change this
test if we change the quality handling.
"""
infile = make_data_path('q1127_src1_grp30.pi')
ui.load_pha(infile)
pha = ui.get_data()
assert pha.quality_filter is None
assert pha.mask is True
assert ui.get_dep().shape == (439, )
assert pha.quality_filter is None
assert pha.mask is True
# The last group is marked as quality=2 and so calling
# ignore_bad means we lose that group.
#
ui.ignore_bad()
assert ui.get_dep().shape == (438, )
assert pha.mask is True
expected = np.ones(1024, dtype=bool)
expected[996:1025] = False
assert pha.quality_filter == pytest.approx(expected)
# What happens when we filter the data? Unlike #1169
# we do change the noticed range.
#
ui.notice(0.5, 7)
assert pha.quality_filter == pytest.approx(expected)
# The mask has been filtered to remove the bad channels
# (this is grouped data)
expected = np.ones(438, dtype=bool)
expected[0:15] = False
expected[410:438] = False
assert pha.mask == pytest.approx(expected)
expected = np.ones(996, dtype=bool)
expected[0:34] = False
expected[481:996] = False
assert pha.get_mask() == pytest.approx(expected)
def test_filter_bad_ungrouped(make_data_path, clean_astro_ui):
"""Check behavior when the data is ungrouped.
This is a test of the current behavior, to check that
values still hold. It may be necessary to change this
test if we change the quality handling.
"""
infile = make_data_path('q1127_src1_grp30.pi')
ui.load_pha(infile)
pha = ui.get_data()
assert pha.quality_filter is None
assert pha.mask is True
assert ui.get_dep().shape == (439, )
ui.ungroup()
assert ui.get_dep().shape == (1024, )
assert pha.quality_filter is None
assert pha.mask is True
ui.ignore_bad()
assert ui.get_dep().shape == (1024, )
assert pha.quality_filter is None
expected = np.ones(1024, dtype=bool)
expected[996:1025] = False
assert pha.mask == pytest.approx(expected)
# At this point we've changed the mask array so Sherpa thinks
# we've applied a filter, so a notice is not going to change
# anything. See issue #1169
#
ui.notice(0.5, 7)
assert pha.mask == pytest.approx(expected)
# We need to ignore to change the mask.
#
ui.ignore(None, 0.5)
ui.ignore(7, None)
expected[0:35] = False
expected[479:1025] = False
assert pha.mask == pytest.approx(expected)
def validate_xspec_result(l, h, npts, ndof, statval):
"""Check that the stat results match those from XSPEC.
This assumes the first data set returned by get_stat_info
should be used.
"""
ui.notice(None, None)
ui.ignore(None, l)
ui.ignore(h, None)
ui.ignore_bad()
sinfo = ui.get_stat_info()[0]
assert sinfo.numpoints == npts
assert sinfo.dof == ndof
# XSPEC displays results to ~ 2dp, so the tolerance
# is quite forgiving here. Or I could use pyxspec to
# calculate and display this.
#
assert_allclose(sinfo.statval, statval, rtol=0, atol=0.005)
def validate_xspec_result(l, h, npts, ndof, statval):
"""Check that the stat results match those from XSPEC.
This assumes the first data set returned by get_stat_info
should be used.
"""
ui.notice(None, None)
ui.ignore(None, l)
ui.ignore(h, None)
ui.ignore_bad()
sinfo = ui.get_stat_info()[0]
assert sinfo.numpoints == npts
assert sinfo.dof == ndof
# XSPEC displays results to ~ 2dp, so the tolerance
# is quite forgiving here. Or I could use pyxspec to
# calculate and display this.
#
assert_allclose(sinfo.statval, statval, rtol=0, atol=0.005)
def test_filter_basic(make_data_path):
"""Test out issue 361 without ignore_bad calls.
This should not trigger the bug behind issue 361.
"""
stats = setup_model(make_data_path)
s1 = ui.calc_stat()
assert s1 == pytest.approx(stats['all'])
ui.ignore_bad()
s2 = ui.calc_stat()
assert s2 == pytest.approx(stats['bad'])
ui.notice(None, None)
s3 = ui.calc_stat()
assert s3 == pytest.approx(stats['all'])
ui.notice(0.5, 8.0)
s4 = ui.calc_stat()
assert s4 == pytest.approx(stats['0.5-8.0'])
def check_bad_grouping(exp_xlo, exp_xhi, exp_counts, lo1, hi1, lo2, hi2):
"""Common tests from test_grouped_pha_all_badXXX
Sending in two ranges is a bit excessive but easiest
thing to implement
"""
cts = ui.get_counts()
assert cts == pytest.approx([exp_counts])
dplot = ui.get_data_plot()
assert dplot.xlo == pytest.approx([exp_xlo])
assert dplot.xhi == pytest.approx([exp_xhi])
assert dplot.y == pytest.approx([exp_counts])
# ignore all the data
ui.ignore(lo1, hi1)
# can still plot
cts = ui.get_counts()
assert cts == pytest.approx([exp_counts])
cts = ui.get_counts(filter=True)
assert len(cts) == 0
dplot = ui.get_data_plot()
assert len(dplot.xlo) == 0
assert len(dplot.xhi) == 0
assert len(dplot.y) == 0
# ignore does not fail
#
ui.ignore(lo2, hi2)
# we can restore the data
ui.notice(None, None)
cts = ui.get_counts()
assert cts == pytest.approx([exp_counts])
dplot = ui.get_data_plot()
assert dplot.xlo == pytest.approx([exp_xlo])
assert dplot.xhi == pytest.approx([exp_xhi])
assert dplot.y == pytest.approx([exp_counts])
# now ignore the bad channels (ie everything)
#
ui.ignore_bad()
cts = ui.get_counts()
assert len(cts) == 0
dplot = ui.get_data_plot()
assert len(dplot.xlo) == 0
assert len(dplot.xhi) == 0
assert len(dplot.y) == 0
# there's nothing to notice (this line is an example of #790)
ui.notice(lo1, hi1)
cts = ui.get_counts()
assert len(cts) == 0
dplot = ui.get_data_plot()
assert len(dplot.xlo) == 0
assert len(dplot.xhi) == 0
assert len(dplot.y) == 0
