def setUpClass(cls) -> None:
# Just to make sure we are running some mongo server, see test-class docstring
cls.test_run_ids = ['0', '1']
cls.all_targets = ('peaks', 'records')
uri = os.environ.get('TEST_MONGO_URI')
db_name = 'test_rundb'
cls.collection_name = 'test_rundb_coll'
client = pymongo.MongoClient(uri)
cls.database = client[db_name]
collection = cls.database[cls.collection_name]
cls.path = os.path.join(tempfile.gettempdir(), 'strax_data')
# assert cls.collection_name not in cls.database.list_collection_names()
if not straxen.utilix_is_configured():
# Bit of an ugly hack but there is no way to get around this
# function even though we don't need it
straxen.rundb.utilix.rundb.xent_collection = lambda *args, **kwargs: collection
cls.rundb_sf = straxen.RunDB(
readonly=False,
runid_field='number',
new_data_path=cls.path,
minimum_run_number=-1,
rucio_path='./strax_test_data',
)
cls.rundb_sf.client = client
cls.rundb_sf.collection = collection
# Extra test for regexes
class RunDBTestLocal(straxen.RunDB):
"""Change class to mathc current host too"""
hosts = {'bla': f'{socket.getfqdn()}'}
cls.rundb_sf_with_current_host = RunDBTestLocal(
readonly=False,
runid_field='number',
new_data_path=cls.path,
minimum_run_number=-1,
rucio_path='./strax_test_data',
)
cls.rundb_sf_with_current_host.client = client
cls.rundb_sf_with_current_host.collection = collection
cls.st = strax.Context(
register=[Records, Peaks],
storage=[cls.rundb_sf],
use_per_run_defaults=False,
config=dict(bonus_area=0),
)
def test_nT(ncores=1):
if ncores == 1:
print('-- nT lazy mode --')
st = straxen.contexts.xenonnt_online(
_database_init=straxen.utilix_is_configured())
offline_gain_model = ("to_pe_placeholder", True)
_update_context(st, ncores, fallback_gains=offline_gain_model, nt=True)
# Lets take an abandoned run where we actually have gains for in the CMT
_run_plugins(st, make_all=True, max_wokers=ncores, run_id=test_run_id_nT)
# Test issue #233
st.search_field('cs1')
# Test of child plugins:
_test_child_options(st)
print(st.context_config)
def test_online_monitor(target='online_peak_monitor', max_tries=3):
"""
See if we can get some data from the online monitor before max_tries
:param target: target to test getting from the online monitor
:param max_tries: number of queries max allowed to get a non-failing
run
"""
if not straxen.utilix_is_configured():
warn('Cannot test online monitor because utilix is not configured')
return
st = straxen.contexts.xenonnt_online()
om = _patch_om_init(target)
st.storage = [om]
max_run = None
for i in range(max_tries):
query = {'provides_meta': True, 'data_type': target}
if max_run is not None:
# One run failed before, lets try a more recent one.
query.update({'number': {"$gt": int(max_run)}})
some_run = om.db[om.col_name].find_one(query,
projection={
'number': 1,
'metadata': 1,
'lineage_hash': 1,
})
if some_run.get('lineage_hash', False):
if some_run['lineage_hash'] != st.key_for("0",
target).lineage_hash:
# We are doing a new release, therefore there is no
# matching data. This makes sense.
return
if some_run is None or some_run.get('number', None) is None:
print(f'Found None')
continue
elif 'exception' in some_run.get('metadata', {}):
# Did find a run, but it is bad, we need to find another one
print(f'Found {some_run.get("number", "No number")} with errors')
max_run = some_run.get("number", -1)
continue
else:
# Correctly written
run_id = f'{some_run["number"]:06}'
break
else:
raise FileNotFoundError(f'No non-failing {target} found in the online '
f'monitor after {max_tries}. Looked for:\n'
f'{st.key_for("0", target)}')
st.get_array(run_id, target, seconds_range=(0, 1), allow_incomplete=True)
def set_script(cls):
if not straxen.utilix_is_configured():
return
# setting up instructions like this may take a while. You can set e.g.
instructions = dict(
event_rate=50, # Don't make too large -> overlapping truth info
chunk_size=1, # keep large -> less overhead but takes more RAM
n_chunk=2,
tpc_radius=straxen.tpc_r,
tpc_length=straxen.tpc_z,
drift_field=200, # kV/cm
energy_range=[2, 10], # keV
nest_inst_types=wfsim.NestId.ER,
)
temp_dir = cls.tempdir
instructions_csv = os.path.join(temp_dir, 'inst.csv')
pema.inst_to_csv(instructions_csv,
get_inst_from=pema.rand_instructions,
**instructions)
config_update = {
"detector": 'XENONnT',
"fax_file": os.path.abspath(instructions_csv),
"cmt_run_id_sim": run_id,
}
print("Temporary directory is ", temp_dir)
os.chdir(temp_dir)
st = pema.pema_context(
fax_config='fax_config_nt_sr0_v0.json',
cmt_run_id_sim=run_id,
base_dir=temp_dir,
raw_dir=temp_dir,
data_dir=temp_dir,
config_update=config_update,
)
st.set_context_config({
'allow_shm': True,
'allow_lazy': False,
'timeout': 300,
'max_messages': 10,
})
script_writer = pema.ProcessRun(
st, run_id, ('raw_records', 'records', 'peaklets', 'truth_matched',
'match_acceptance_extended'))
cls.script = script_writer
def test_1T_elife():
"""
Test elife from CMT DB against historical data(aux file)
"""
if not straxen.utilix_is_configured():
warn('Cannot do test becaus ' 'no have access to the database.')
return
elife_conf = ('elife', 'ONLINE', False)
elife_cmt = straxen.get_correction_from_cmt(test_run_id_1T, elife_conf)
elife_file = elife_conf = aux_repo + '3548132b55f81a43654dba5141366041e1daaf01/strax_files/elife.npy'
x = straxen.get_resource(elife_file, fmt='npy')
run_index = np.where(x['run_id'] == int(test_run_id_1T))[0]
elife = x[run_index[0]]['e_life']
mes = 'Elife values do not match. Please check'
assert elife_cmt == elife, mes
def test_nT(ncores=1):
if ncores == 1:
print('-- nT lazy mode --')
init_database = straxen.utilix_is_configured(warning_message=False)
st = straxen.test_utils.nt_test_context(
_database_init=init_database,
use_rucio=False,
)
_update_context(st, ncores, nt=True)
# Lets take an abandoned run where we actually have gains for in the CMT
_run_plugins(st, make_all=True, max_workers=ncores, run_id=nt_test_run_id)
# Test issue #233
st.search_field('cs1')
# Test of child plugins:
_test_child_options(st, nt_test_run_id)
print(st.context_config)
def test_event_display_simple(self):
plot_all_positions = straxen.utilix_is_configured()
with self.assertRaises(NotImplementedError):
# old way of calling the simple display
self.st.event_display_simple(
nt_test_run_id,
time_within=self.first_event,
)
# New, correct way of calling the simple display
self.st.event_display(
nt_test_run_id,
time_within=self.first_event,
xenon1t=False,
plot_all_positions=plot_all_positions,
simple_layout=True,
)
def _extract_latest_comment(
context,
test_for_target='raw_records',
**context_kwargs,
):
if context == 'xenonnt_online' and not straxen.utilix_is_configured():
return
st = getattr(straxen.contexts, context)(**context_kwargs)
assert hasattr(st, 'extract_latest_comment'
), "extract_latest_comment not added to context?"
st.extract_latest_comment()
assert st.runs is not None, "No registry build?"
assert 'comments' in st.runs.keys()
st.select_runs(available=test_for_target)
if context == 'demo':
assert len(st.runs)
assert f'{test_for_target}_available' in st.runs.keys()
def test_target_peaks(self, target='peak_basics', tol=2):
"""
Not a real mini analysis but let's see if the number of peaks
matches some pre-defined value. This is just to safeguard one
from accidentally adding some braking code.
"""
self.assertTrue(target
in self._expected_test_results,
f'No expectation for {target}?!')
data = self.st.get_array(nt_test_run_id, target)
message = (f'Got more/less data for {target}. If you changed something '
f'on {target}, please update the numbers in '
f'tests/test_mini_analyses.TestMiniAnalyses._expected_test_results')
if not straxen.utilix_is_configured():
# If we do things with dummy maps, things might be slightly different
tol += 10
self.assertTrue(np.abs(len(data) - self._expected_test_results[target]) < tol, message)
def get_delay(run_id):
"""
Function which returns the total delay between TPC and veto for a
given run_id. Returns nan if
"""
try:
import utilix
except ModuleNotFoundError:
return np.nan
delay = np.nan
if straxen.utilix_is_configured():
run_db = utilix.DB()
run_meta = run_db.get_doc(run_id)
delay = _get_delay(run_meta)
return delay
def _get_daq_config(
run_id: str,
config_name: str = 'daq_config',
run_collection: typing.Optional[pymongo.collection.Collection] = None) -> dict:
"""
Query the runs database for the config of the daq during this run.
Either use the context of the runs collection.
"""
if run_collection is None:
if not straxen.utilix_is_configured():
raise NotImplementedError('Only works with the runs-database')
run_collection = utilix.rundb.xent_collection()
daq_doc = run_collection.find_one({"number": int(run_id)},
projection={config_name: 1})
if daq_doc is None or config_name not in daq_doc:
raise ValueError(f'Requested {config_name} does not exist')
return daq_doc[config_name]
def test_n_hits():
if not straxen.utilix_is_configured():
return
records = np.zeros(2, dtype=strax.record_dtype())
records['length'] = 5
records['pulse_length'] = 5
records['dt'] = 1
records['channel'] = [0, 1]
records['data'][0, :5] = [0, 1, 1, 0, 1]
records['data'][1, :5] = [0, 1, 0, 0, 0]
st = straxen.contexts.xenonnt_online()
st.set_config({'hit_min_amplitude': 1})
p = st.get_single_plugin('0', 'peaklets')
res = p.compute(records, 0, 999)
peaklets = res['peaklets']
assert peaklets['n_hits'] == 3, f"Peaklet has the wrong number of hits!"
def test_sim_nT_advanced():
"""Test the nT simulator. Works only if one has access to the XENONnT databases.
Clone the repo to dali and type 'pytest' to run. The first run will test simple s1,
garfield s2 and noise/afterpulses. The second run will test the s1 spline model"""
if not straxen.utilix_is_configured():
log.warning(
f"Utilix is not configured, skipping database-requiring tests!")
return
with tempfile.TemporaryDirectory() as tempdir:
log.debug(f'Working in {tempdir}')
st = straxen.contexts.xenonnt_simulation(
cmt_run_id_sim='010000',
cmt_version='global_ONLINE',
_config_overlap={},
)
st.set_config(
dict(gain_model_mc=("to_pe_placeholder", True),
gain_model=("to_pe_placeholder", True),
hit_min_amplitude='pmt_commissioning_initial'))
st.set_config(dict(
nchunk=1,
event_rate=1,
chunk_size=2,
))
st.set_config({
'fax_config_override':
dict(
s2_luminescence_model='simple',
s1_lce_correction_map=
'XENONnT_s1_xyz_LCE_corrected_qes_MCva43fa9b_wires.json.gz',
s1_time_spline=
'XENONnT_s1_proponly_va43fa9b_wires_20200625.json.gz',
s1_model_type='optical_propagation+simple',
)
})
log.debug(f'Getting raw-records')
rr = st.get_array(run_id, 'raw_records')
log.debug(f'Getting peaks')
p = st.get_array(run_id, 'peaks')
_sanity_check(rr, p)
log.info(f'All done')
def resources_available(self):
"""
Exception to skip test if external requirements are not met. Otherwise define
Scada interface as self.sc.
"""
if not straxen.utilix_is_configured(
'scada',
'scdata_url',
):
self.skipTest(
"Cannot test scada since we have no access to xenon secrets.)")
try:
self.sc = straxen.SCADAInterface(use_progress_bar=False)
self.sc.get_new_token()
except requests.exceptions.SSLError:
self.skipTest(
"Cannot reach database since HTTPs certifcate expired.")
def nt_test_context(target_context='xenonnt_online',
deregister=('peak_veto_tags', 'events_tagged'),
**kwargs):
st = getattr(straxen.contexts, target_context)(**kwargs)
st.set_config({'diagnose_sorting': True})
st._plugin_class_registry['raw_records'].__version__ = "MOCKTESTDATA" # noqa
st.storage = [strax.DataDirectory('./strax_test_data')]
download_test_data('https://raw.githubusercontent.com/XENONnT/strax_auxiliary_files/f0d177401e11408b273564f0e29df77528e83d26/strax_files/012882-raw_records-z7q2d2ye2t.tar') # noqa
assert st.is_stored(nt_test_run_id, 'raw_records'), os.listdir(st.storage[-1].path)
to_remove = list(deregister)
if not straxen.utilix_is_configured(warning_message=False):
st.set_config(_testing_config_nT)
to_remove += 'peak_positions_mlp peak_positions_cnn peak_positions_gcn s2_recon_pos_diff'.split() # noqa
# The test data for this plugin doesn't work
to_remove += ['event_pattern_fit']
st.set_config({'gain_model': ("to_pe_placeholder", True)})
st.register(straxen.PeakPositions1T)
for plugin in to_remove:
del st._plugin_class_registry[plugin]
return st
def test_mc_wrapper_gains(
run_id='009000',
cmt_id='016000',
mc_id='mc_0',
execute=True,
):
"""
Test that for two different run ids, we get different gains using
the MC wrapper.
:param run_id: First run-id (used for normal query)
:param cmt_id: Second run-id used as a CMT id (should not be the
same as run_id! otherwise the values might actually be the same
and the test does not work).
:param execute: Execute this test (this is set to False since the
test takes 9 minutes which is too long. We can activate this if
the testing time due to faster CMT queries is reduced).
:return: None
"""
if not straxen.utilix_is_configured() or not execute:
return
assert np.abs(int(run_id) - int(cmt_id)) > 500, 'runs must be far apart'
# First for the run-id let's get the value
gains = straxen.get_correction_from_cmt(run_id,
('to_pe_model', 'ONLINE', True))
# Now, we repeat the same query using the MC wrapper, this should
# give us a different result since we are now asking for a very
# different run-number.
mc_gains_diff = straxen.get_correction_from_cmt(
mc_id, ('MC', cmt_id, 'to_pe_model', 'ONLINE', True))
# Repeat the query from above to verify, let's see if we are getting
# the same results as for `gains` above
mc_gains_same = straxen.get_correction_from_cmt(
mc_id, ('MC', run_id, 'to_pe_model', 'ONLINE', True))
assert not np.all(gains == mc_gains_diff)
assert np.all(gains == mc_gains_same)
def nt_test_context(target_context='xenonnt_online',
deregister=(),
keep_default_storage=False,
**kwargs) -> strax.Context:
"""
Get a dummy context with full nt-like data simulated data (except aqmon)
to allow testing plugins
:param target_context: Which contexts from straxen.contexts to test
:param deregister: a list of plugins from the context
:param keep_default_storage: if to True, keep the default context
storage. Usually, you don't need this since all the data will be
stored in a separate test data folder.
:param kwargs: Any kwargs are passed to the target-context
:return: a context
"""
if not straxen.utilix_is_configured(warning_message=False):
kwargs.setdefault('_database_init', False)
st = getattr(straxen.contexts, target_context)(**kwargs)
st.set_config({'diagnose_sorting': True, 'store_per_channel': True})
st.register(_get_fake_daq_reader())
download_test_data('https://raw.githubusercontent.com/XENONnT/'
'strax_auxiliary_files/'
'f0d177401e11408b273564f0e29df77528e83d26/'
'strax_files/'
'012882-raw_records-z7q2d2ye2t.tar')
if keep_default_storage:
st.storage += [strax.DataDirectory('./strax_test_data')]
else:
st.storage = [strax.DataDirectory('./strax_test_data')]
assert st.is_stored(nt_test_run_id,
'raw_records'), os.listdir(st.storage[-1].path)
to_remove = list(deregister)
for plugin in to_remove:
del st._plugin_class_registry[plugin]
return st
def setUpClass(cls) -> None:
"""
For testing purposes, slightly alter the RucioFrontend such that
we can run tests outside of dali too
"""
if not straxen.utilix_is_configured():
return
if 'rcc' not in socket.getfqdn():
# If we are not on RCC, for testing, add some dummy site
straxen.RucioFrontend.local_rses = {
'UC_DALI_USERDISK': r'.rcc.',
'test_rucio': f'{socket.getfqdn()}'
}
straxen.RucioFrontend.get_rse_prefix = lambda *x: 'test_rucio'
# Some non-existing keys that we will try finding in the test cases.
cls.test_keys = [
strax.DataKey(run_id=run_id,
data_type='dtype',
lineage={
'dtype': ['Plugin', '0.0.0.', {}],
}) for run_id in ('-1', '-2')
]
def test_ext_timings_nv(records):
"""
Little test for nVetoExtTimings.
"""
if not straxen.utilix_is_configured():
return
# several fake records do not have any pulse length
# and channel start at 0, convert to nv:
records['pulse_length'] = records['length']
records['channel'] += 2000
st = straxen.contexts.xenonnt_led()
plugin = st.get_single_plugin('1', 'ext_timings_nv')
hits = strax.find_hits(records, min_amplitude=1)
hitlets = np.zeros(len(hits), strax.hitlet_dtype())
strax.copy_to_buffer(hits, hitlets, '_refresh_hit_to_hitlets')
result = plugin.compute(hitlets, records)
assert len(result) == len(hits)
assert np.all(result['time'] == hits['time'])
assert np.all(result['pulse_i'] == hits['record_i'])
true_dt = hits['time'] - records[hits['record_i']]['time']
assert np.all(result['delta_time'] == true_dt)
class TestItpMaps(TestCase):
def open_map(self, map_name, fmt, method='WeightedNearestNeighbors'):
map_data = get_resource(map_name, fmt=fmt)
m = InterpolatingMap(map_data, method=method)
self.assertTrue(m is not None)
@skipIf(not utilix_is_configured(),
'Cannot download maps without db access')
def test_lce_map(self):
self.open_map(
'XENONnT_s1_xyz_LCE_corrected_qes_MCva43fa9b_wires.json.gz',
fmt='json.gz')
def test_array_valued(self):
"""See https://github.com/XENONnT/straxen/pull/757"""
_map = {
'coordinate_system': [[-18.3, -31.7, -111.5], [36.6, -0.0, -111.5],
[-18.3, 31.7, -111.5], [-18.3, -31.7, -37.5],
[36.6, -0.0, -37.5], [-18.3, 31.7, -37.5]],
'description':
'Array_valued dummy map with lists',
'name':
'Dummy map',
'map': [[1.7, 11.1], [1.7, 11.1], [1.7, 11.0], [3.3, 5.7],
[3.3, 5.8], [3.3, 5.7]]
}
itp_map = InterpolatingMap(_map)
# Let's do something easy, check if one fixed point yields the
# same result if not, our interpolation map depends on the
# straxen version?! That's bad!
map_at_random_point = itp_map([[0, 0, 0], [0, 0, -140]])
self.assertAlmostEqual(map_at_random_point[0][0], 2.80609655)
self.assertAlmostEqual(map_at_random_point[0][1], 7.37967879)
self.assertAlmostEqual(map_at_random_point[1][0], 2.17815179)
self.assertAlmostEqual(map_at_random_point[1][1], 9.47282782)
class SimpleTests(TestCase):
"""Odd bunch of tests that can be removed if needed"""
def check_raises_check_args(self):
truth = np.zeros(1, dtype=[(('bla bla', 'bla'), np.int8)])
with self.assertRaises(ValueError):
pema.compare_plots._check_args(truth, run_id=None)
with self.assertRaises(ValueError):
pema.compare_plots._check_args([],
truth_vs_custom=truth,
run_id='bla')
def test_show(self):
pema.compare_plots._save_and_show('a', '.test', show=True, peak_i=1)
def test_warnings_for_matching(self):
with self.assertRaises(ValueError):
peak1 = np.zeros(1, dtype=[(('bla bla', 'bla'), np.int8)])
peak2 = peak1
pema.match_peaks(peak1, peak2)
@skipIf(not straxen.utilix_is_configured(), 'no utilix file')
def test_context(self):
with self.assertRaises(FileNotFoundError):
pema.pema_context(base_dir='/the/moon/path',
fax_config='bla',
cmt_run_id_sim='1')
with self.assertRaises(ValueError):
pema.pema_context(base_dir='./',
config_update=1,
fax_config='bla',
cmt_run_id_sim='1')
with self.assertRaises(RuntimeError):
pema.pema_context(base_dir='./',
config_update={},
fax_config='fax_config_nt_design.json',
cmt_run_id_sim='1')
def test_connect_to_db():
"""
Test connection to db
"""
if not straxen.utilix_is_configured():
warn('Cannot do test becaus ' 'no have access to the database.')
return
username = None
password = None
mongo_url = None
is_nt = True
mongo_kwargs = {
'url': mongo_url,
'user': username,
'password': password,
'database': 'corrections'
}
corrections_collection = utilix.rundb.xent_collection(**mongo_kwargs)
client = corrections_collection.database.client
cmt = strax.CorrectionsInterface(client, database_name='corrections')
df = cmt.read('global')
mes = 'Return empty dataframe when reading DB. Please check'
assert not df.empty, mes
def test_sim_nT():
"""Test the nT simulator. Works only if one has access to the XENONnT databases"""
if not straxen.utilix_is_configured():
# This means we cannot load the nT files. Most likely will work
# locally but not a travis job.
return
with tempfile.TemporaryDirectory() as tempdir:
log.debug(f'Working in {tempdir}')
st = strax.Context(storage=tempdir,
config=dict(nchunk=1,
event_rate=1,
chunk_size=2,
detector='XENONnT',
fax_config='fax_config_nt.json',
**straxen.contexts.xnt_common_config),
**straxen.contexts.common_opts)
st.register(wfsim.RawRecordsFromFaxNT)
log.debug(f'Getting raw-records')
rr = st.get_array(run_id, 'raw_records')
log.debug(f'Getting peaks')
p = st.get_array(run_id, 'peaks')
_sanity_check(rr, p)
log.info(f'All done')
def _update_context(st, max_workers, nt=True):
# Ignore strax-internal warnings
st.set_context_config({'free_options': tuple(st.config.keys())})
if not nt:
st.register(DummyRawRecords)
if straxen.utilix_is_configured(warning_message=False):
# Set some placeholder gain as this takes too long for 1T to load from CMT
st.set_config({
k: v
for k, v in testing_config_1T.items()
if k in ('hev_gain_model', 'gain_model')
})
else:
st.set_config(testing_config_1T)
if max_workers - 1:
st.set_context_config({
'allow_multiprocess': True,
'allow_lazy': False,
'timeout': 60, # we don't want to build travis for ever
})
print('--- Plugins ---')
for k, v in st._plugin_class_registry.items():
print(k, v)
def test_query_sc_values():
"""
Unity test for the SCADAInterface. Query a fixed range and check if
return is correct.
"""
if not straxen.utilix_is_configured('scada', 'scdata_url'):
warnings.warn(
'Cannot test scada since we have no access to xenon secrets.')
return
print('Testing SCADAInterface')
sc = straxen.SCADAInterface(use_progress_bar=False)
print('Query single value:')
# Simple query test:
# Query 5 s of data:
start = 1609682275000000000
end = start + 5 * 10**9
parameters = {'SomeParameter': 'XE1T.CTPC.Board06.Chan011.VMon'}
df = sc.get_scada_values(
parameters,
start=start,
end=end,
every_nth_value=1,
query_type_lab=False,
)
assert df['SomeParameter'][
0] // 1 == 1253, 'First values returned is not corrrect.'
assert np.all(np.isnan(
df['SomeParameter'][1:])), 'Subsequent values are not correct.'
# Test ffill option:
print('Testing forwardfill option:')
parameters = {'SomeParameter': 'XE1T.CRY_FCV104FMON.PI'}
df = sc.get_scada_values(
parameters,
start=start,
end=end,
fill_gaps='forwardfill',
every_nth_value=1,
query_type_lab=False,
)
assert np.all(np.isclose(
df[:4], 2.079859)), 'First four values deviate from queried values.'
assert np.all(np.isclose(
df[4:], 2.117820)), 'Last two values deviate from queried values.'
print('Testing downsampling and averaging option:')
parameters = {'SomeParameter': 'XE1T.CRY_TE101_TCRYOBOTT_AI.PI'}
df_all = sc.get_scada_values(
parameters,
start=start,
end=end,
fill_gaps='forwardfill',
every_nth_value=1,
query_type_lab=False,
)
df = sc.get_scada_values(
parameters,
start=start,
end=end,
fill_gaps='forwardfill',
down_sampling=True,
every_nth_value=2,
query_type_lab=False,
)
assert np.all(
df_all[::2] == df), 'Downsampling did not return the correct values.'
df = sc.get_scada_values(
parameters,
start=start,
end=end,
fill_gaps='forwardfill',
every_nth_value=2,
query_type_lab=False,
)
# Compare average for each second value:
for ind, i in enumerate([0, 2, 4]):
assert np.isclose(np.mean(df_all[i:i + 2]),
df['SomeParameter'][ind]), 'Averaging is incorrect.'
# Testing lab query type:
df = sc.get_scada_values(
parameters,
start=start,
end=end,
query_type_lab=True,
)
assert np.all(df['SomeParameter'] //
1 == -96), 'Not all values are correct for query type lab.'
def xenonnt_simulation(output_folder='./strax_data',
wfsim_registry='RawRecordsFromFaxNT',
cmt_run_id_sim=None,
cmt_run_id_proc=None,
cmt_version='global_v5',
fax_config='fax_config_nt_design.json',
overwrite_from_fax_file_sim=False,
overwrite_from_fax_file_proc=False,
cmt_option_overwrite_sim=immutabledict(),
cmt_option_overwrite_proc=immutabledict(),
_forbid_creation_of=None,
_config_overlap=immutabledict(
drift_time_gate='electron_drift_time_gate',
drift_velocity_liquid='electron_drift_velocity',
electron_lifetime_liquid='elife'),
**kwargs):
"""
The most generic context that allows for setting full divergent
settings for simulation purposes
It makes full divergent setup, allowing to set detector simulation
part (i.e. for wfsim up to truth and raw_records). Parameters _sim
refer to detector simulation parameters.
Arguments having _proc in their name refer to detector parameters that
are used for processing of simulations as done to the real datector
data. This means starting from already existing raw_records and finishing
with higher level data, such as peaks, events etc.
If only one cmt_run_id is given, the second one will be set automatically,
resulting in CMT match between simulation and processing. However, detector
parameters can be still overwritten from fax file or manually using cmt
config overwrite options.
CMT options can also be overwritten via fax config file.
:param output_folder: Output folder for strax data.
:param wfsim_registry: Name of WFSim plugin used to generate data.
:param cmt_run_id_sim: Run id for detector parameters from CMT to be used
for creation of raw_records.
:param cmt_run_id_proc: Run id for detector parameters from CMT to be used
for processing from raw_records to higher level data.
:param cmt_version: Global version for corrections to be loaded.
:param fax_config: Fax config file to use.
:param overwrite_from_fax_file_sim: If true sets detector simulation
parameters for truth/raw_records from from fax_config file istead of CMT
:param overwrite_from_fax_file_proc: If true sets detector processing
parameters after raw_records(peaklets/events/etc) from from fax_config
file istead of CMT
:param cmt_option_overwrite_sim: Dictionary to overwrite CMT settings for
the detector simulation part.
:param cmt_option_overwrite_proc: Dictionary to overwrite CMT settings for
the data processing part.
:param _forbid_creation_of: str/tuple, of datatypes to prevent form
being written (e.g. 'raw_records' for read only simulation context).
:param _config_overlap: Dictionary of options to overwrite. Keys
must be simulation config keys, values must be valid CMT option keys.
:param kwargs: Additional kwargs taken by strax.Context.
:return: strax.Context instance
"""
import wfsim
st = strax.Context(storage=strax.DataDirectory(output_folder),
config=dict(
detector='XENONnT',
fax_config=fax_config,
check_raw_record_overlaps=True,
**straxen.contexts.xnt_common_config,
),
**straxen.contexts.xnt_common_opts,
**kwargs)
st.register(getattr(wfsim, wfsim_registry))
# Make sure that the non-simulated raw-record types are not requested
st.deregister_plugins_with_missing_dependencies()
if straxen.utilix_is_configured(
warning_message='Bad context as we cannot set CMT since we '
'have no database access'
''):
st.apply_cmt_version(cmt_version)
if _forbid_creation_of is not None:
st.context_config['forbid_creation_of'] += strax.to_str_tuple(
_forbid_creation_of)
# doing sanity checks for cmt run ids for simulation and processing
if (not cmt_run_id_sim) and (not cmt_run_id_proc):
raise RuntimeError("cmt_run_id_sim and cmt_run_id_proc are None. "
"You have to specify at least one CMT run id. ")
if (cmt_run_id_sim
and cmt_run_id_proc) and (cmt_run_id_sim != cmt_run_id_proc):
print("INFO : divergent CMT runs for simulation and processing")
print(" cmt_run_id_sim".ljust(25), cmt_run_id_sim)
print(" cmt_run_id_proc".ljust(25), cmt_run_id_proc)
else:
cmt_id = cmt_run_id_sim or cmt_run_id_proc
cmt_run_id_sim = cmt_id
cmt_run_id_proc = cmt_id
# Replace default cmt options with cmt_run_id tag + cmt run id
cmt_options = straxen.get_corrections.get_cmt_options(st)
# prune to just get the strax options
cmt_options = {
key: val['strax_option']
for key, val in cmt_options.items()
}
# First, fix gain model for simulation
st.set_config({
'gain_model_mc':
('cmt_run_id', cmt_run_id_sim, *cmt_options['gain_model'])
})
fax_config_override_from_cmt = dict()
for fax_field, cmt_field in _config_overlap.items():
fax_config_override_from_cmt[fax_field] = ('cmt_run_id',
cmt_run_id_sim,
*cmt_options[cmt_field])
st.set_config(
{'fax_config_override_from_cmt': fax_config_override_from_cmt})
# and all other parameters for processing
for option in cmt_options:
st.config[option] = ('cmt_run_id', cmt_run_id_proc,
*cmt_options[option])
# Done with "default" usage, now to overwrites from file
#
# Take fax config and put into context option
if overwrite_from_fax_file_proc or overwrite_from_fax_file_sim:
fax_config = straxen.get_resource(fax_config, fmt='json')
for fax_field, cmt_field in _config_overlap.items():
if overwrite_from_fax_file_proc:
st.config[cmt_field] = (cmt_options[cmt_field][0] +
'_constant', fax_config[fax_field])
if overwrite_from_fax_file_sim:
st.config['fax_config_override_from_cmt'][fax_field] = (
cmt_options[cmt_field][0] + '_constant',
fax_config[fax_field])
# And as the last step - manual overrrides, since they have the highest priority
# User customized for simulation
for option in cmt_option_overwrite_sim:
if option not in cmt_options:
raise ValueError(
f'Overwrite option {option} is not using CMT by default '
'you should just use set config')
if option not in _config_overlap.values():
raise ValueError(
f'Overwrite option {option} does not have mapping from '
f'CMT to fax config!')
for fax_key, cmt_key in _config_overlap.items():
if cmt_key == option:
_name_index = 2 if 'cmt_run_id' in cmt_options[option] else 0
st.config['fax_config_override_from_cmt'][fax_key] = (
cmt_options[option][_name_index] + '_constant',
cmt_option_overwrite_sim[option])
del _name_index
del (fax_key, cmt_key)
# User customized for simulation
for option in cmt_option_overwrite_proc:
if option not in cmt_options:
raise ValueError(
f'Overwrite option {option} is not using CMT by default '
'you should just use set config')
_name_index = 2 if 'cmt_run_id' in cmt_options[option] else 0
st.config[option] = (cmt_options[option][_name_index] + '_constant',
cmt_option_overwrite_proc[option])
del _name_index
# Only for simulations
st.set_config({"event_info_function": "disabled"})
return st
class TestMiniAnalyses(unittest.TestCase):
"""
Generally, tests in this class run st.<some_mini_analysis>
We provide minimal arguments to just probe if the
<some_mini_analysis> is not breaking when running, we are NOT
checking if plots et cetera make sense, just if the code is not
broken (e.g. because for changes in dependencies like matplotlib or
bokeh)
NB! If this tests fails locally (but not on github-CI), please do:
`rm strax_test_data`
You might be an old version of test data.
"""
# They were added on 25/10/2021 and may be outdated by now
_expected_test_results = {
'peak_basics': 40,
'n_s1': 19,
'run_live_time': 4.7516763,
'event_basics': 20,
}
@classmethod
def setUpClass(cls) -> None:
"""
Common setup for all the tests. We need some data which we
don't delete but reuse to prevent a lot of computations in this
class
"""
cls.st = nt_test_context()
# For al the WF plotting, we might need records, let's make those
cls.st.make(nt_test_run_id, 'records')
cls.first_peak = cls.st.get_array(nt_test_run_id, 'peak_basics')[0]
cls.first_event = cls.st.get_array(nt_test_run_id, 'event_basics')[0]
def tearDown(self):
"""After each test, clear a figure (if one was open)"""
plt_clf()
def test_target_peaks(self, target='peak_basics', tol=2):
"""
Not a real mini analysis but let's see if the number of peaks
matches some pre-defined value. This is just to safeguard one
from accidentally adding some braking code.
"""
self.assertTrue(target in self._expected_test_results,
f'No expectation for {target}?!')
data = self.st.get_array(nt_test_run_id, target)
message = (
f'Got more/less data for {target}. If you changed something '
f'on {target}, please update the numbers in '
f'tests/test_mini_analyses.TestMiniAnalyses._expected_test_results'
)
if not straxen.utilix_is_configured():
# If we do things with dummy maps, things might be slightly different
tol += 10
self.assertTrue(
np.abs(len(data) - self._expected_test_results[target]) < tol,
message)
def test_target_events(self):
"""Test that the number of events is roughly right"""
self.test_target_peaks(target='event_basics')
def test_plot_waveform(self, deep=False):
self.st.plot_waveform(nt_test_run_id,
time_within=self.first_peak,
deep=deep)
def test_plot_waveform_deep(self):
self.test_plot_waveform(deep=True)
def test_plot_hit_pattern(self):
self.st.plot_hit_pattern(nt_test_run_id,
time_within=self.first_peak,
xenon1t=False)
def test_plot_records_matrix(self):
self._st_attr_for_one_peak('plot_records_matrix')
def test_raw_records_matrix(self):
self._st_attr_for_one_peak('raw_records_matrix')
def test_event_display_simple(self):
plot_all_positions = straxen.utilix_is_configured()
with self.assertRaises(NotImplementedError):
# old way of calling the simple display
self.st.event_display_simple(
nt_test_run_id,
time_within=self.first_event,
)
# New, correct way of calling the simple display
self.st.event_display(
nt_test_run_id,
time_within=self.first_event,
xenon1t=False,
plot_all_positions=plot_all_positions,
simple_layout=True,
)
def test_single_event_plot(self):
plot_all_positions = straxen.utilix_is_configured()
straxen.analyses.event_display.plot_single_event(
self.st,
nt_test_run_id,
events=self.st.get_array(nt_test_run_id, 'events'),
event_number=self.first_event['event_number'],
xenon1t=False,
plot_all_positions=plot_all_positions,
)
def test_event_display_interactive(self):
self.st.event_display_interactive(
nt_test_run_id,
time_within=self.first_event,
xenon1t=False,
)
def test_plot_peaks_aft_histogram(self):
self.st.plot_peaks_aft_histogram(nt_test_run_id)
@unittest.skipIf(not straxen.utilix_is_configured(),
"No db access, cannot test CMT.")
def test_event_scatter(self):
self.st.event_scatter(nt_test_run_id)
def test_event_scatter_diff_options(self):
self.st.event_scatter(nt_test_run_id,
color_range=(0, 10),
color_dim='s1_area')
def test_energy_spectrum(self):
self.st.plot_energy_spectrum(nt_test_run_id)
def test_energy_spectrum_diff_options(self):
"""Run st.plot_energy_spectrum with several options"""
self.st.plot_energy_spectrum(nt_test_run_id,
unit='kg_day_kev',
exposure_kg_sec=1)
self.st.plot_energy_spectrum(nt_test_run_id,
unit='tonne_day_kev',
exposure_kg_sec=1)
self.st.plot_energy_spectrum(nt_test_run_id,
unit='tonne_year_kev',
exposure_kg_sec=1,
geomspace=False)
with self.assertRaises(ValueError):
# Some units shouldn't be allowed
self.st.plot_energy_spectrum(nt_test_run_id,
unit='not_allowed_unit',
exposure_kg_sec=1)
def test_peak_classification(self):
self.st.plot_peak_classification(nt_test_run_id)
def _st_attr_for_one_peak(self, function_name):
"""
Utility function to prevent having to copy past the code
below for all the functions we are going to test for one peak
"""
f = getattr(self.st, function_name)
f(nt_test_run_id, time_within=self.first_peak)
@unittest.skipIf(is_py310(), 'holoviews incompatible with py3.10')
def test_waveform_display(self):
"""test st.waveform_display for one peak"""
self._st_attr_for_one_peak('waveform_display')
def test_hvdisp_plot_pmt_pattern(self):
"""test st.hvdisp_plot_pmt_pattern for one peak"""
self._st_attr_for_one_peak('hvdisp_plot_pmt_pattern')
def test_hvdisp_plot_peak_waveforms(self):
"""test st.hvdisp_plot_peak_waveforms for one peak"""
self._st_attr_for_one_peak('hvdisp_plot_peak_waveforms')
def test_plot_pulses_tpc(self):
"""
Test that we can plot some TPC pulses and fail if raise a
ValueError if an invalid combination of parameters is given
"""
self.st.plot_pulses_tpc(
nt_test_run_id,
time_within=self.first_peak,
max_plots=2,
plot_hits=True,
ignore_time_warning=False,
store_pdf=True,
)
with self.assertRaises(ValueError):
# Raise an error if no time range is specified
self.st.plot_pulses_tpc(
nt_test_run_id,
max_plots=2,
plot_hits=True,
ignore_time_warning=True,
store_pdf=True,
)
def test_plot_pulses_mv(self):
"""Repeat above for mv"""
self.st.plot_pulses_mv(
nt_test_run_id,
max_plots=2,
plot_hits=True,
ignore_time_warning=True,
)
def test_plot_pulses_nv(self):
"""Repeat above for nv"""
self.st.plot_pulses_nv(
nt_test_run_id,
max_plots=2,
plot_hits=True,
ignore_time_warning=True,
)
@unittest.skipIf(not straxen.utilix_is_configured(),
"No db access, cannot test!")
def test_event_display(self):
"""Event display plot, needs CMT"""
self.st.event_display(nt_test_run_id, time_within=self.first_event)
@unittest.skipIf(not straxen.utilix_is_configured(),
"No db access, cannot test!")
def test_event_display_no_rr(self):
"""Make an event display without including records"""
self.st.event_display(
nt_test_run_id,
time_within=self.first_event,
records_matrix=False,
event_time_limit=[
self.first_event['time'], self.first_event['endtime']
],
)
def test_calc_livetime(self):
"""Use straxen.get_livetime_sec"""
try:
live_time = straxen.get_livetime_sec(self.st, nt_test_run_id)
except strax.RunMetadataNotAvailable:
things = self.st.get_array(nt_test_run_id, 'peaks')
live_time = straxen.get_livetime_sec(self.st,
nt_test_run_id,
things=things)
assertion_statement = "Live-time calculation is wrong"
expected = self._expected_test_results['run_live_time']
self.assertTrue(live_time == expected, assertion_statement)
def test_df_wiki(self):
"""We have a nice utility to write dataframes to the wiki"""
df = self.st.get_df(nt_test_run_id, 'peak_basics')[:10]
straxen.dataframe_to_wiki(df)
@unittest.skipIf(straxen.utilix_is_configured(), "Test for no DB access")
def test_daq_plot_errors_without_utilix(self):
"""
We should get a not implemented error if we call a function
in the daq_waveforms analyses
"""
with self.assertRaises(NotImplementedError):
straxen.analyses.daq_waveforms._get_daq_config('som_run',
run_collection=None)
@unittest.skipIf(not straxen.utilix_is_configured(),
"No db access, cannot test!")
def test_daq_plot_errors(self):
"""To other ways we should not be allowed to call daq_waveforms.XX"""
with self.assertRaises(ValueError):
straxen.analyses.daq_waveforms._get_daq_config('no_run')
with self.assertRaises(ValueError):
straxen.analyses.daq_waveforms._board_to_host_link(
{'boards': [{
'no_boards': 0
}]}, 1)
@unittest.skipIf(not straxen.utilix_is_configured(),
"No db access, cannot test!")
def test_event_plot_errors(self):
"""
Several Exceptions should be raised with these following bad
ways of calling the event display
"""
with self.assertRaises(ValueError):
# Wrong way of calling records matrix
self.st.event_display(nt_test_run_id,
records_matrix='records_are_bad')
with self.assertRaises(ValueError):
# A single event should not have three entries
straxen.analyses.event_display._event_display(events=[1, 2, 3],
context=self.st,
to_pe=None,
run_id='1')
with self.assertRaises(ValueError):
# Can't pass empty axes like this to the inner script
straxen.analyses.event_display._event_display(
axes=None,
events=[None],
context=self.st,
to_pe=None,
run_id=nt_test_run_id,
)
with self.assertRaises(ValueError):
# Should raise a valueError
straxen.analyses.event_display.plot_single_event(
context=None,
run_id=None,
events=[1, 2, 3],
event_number=None,
)
with self.assertRaises(ValueError):
# Give to many recs to this inner script
straxen.analyses.event_display._scatter_rec(_event=None,
recs=list(range(10)))
@unittest.skipIf(is_py310(), 'holoviews incompatible with py3.10')
def test_interactive_display(self):
"""Run and save interactive display"""
fig = self.st.event_display_interactive(
nt_test_run_id,
time_within=self.first_event,
xenon1t=False,
plot_record_matrix=True,
)
save_as = 'test_display.html'
fig.save(save_as)
self.assertTrue(os.path.exists(save_as))
os.remove(save_as)
self.assertFalse(os.path.exists(save_as))
st = self.st.new_context()
st.event_display_interactive(
nt_test_run_id,
time_within=self.first_event,
xenon1t=False,
plot_record_matrix=False,
only_main_peaks=True,
)
def test_bokeh_selector(self):
"""Test the bokeh data selector"""
from straxen.analyses.bokeh_waveform_plot import DataSelectionHist
p = self.st.get_array(nt_test_run_id, 'peak_basics')
ds = DataSelectionHist('ds')
fig = ds.histogram2d(p,
p['area'],
p['area'],
bins=50,
hist_range=((0, 200), (0, 2000)),
log_color_scale=True,
clim=(10, None),
undeflow_color='white')
import bokeh.plotting as bklt
save_as = 'test_data_selector.html'
bklt.save(fig, save_as)
self.assertTrue(os.path.exists(save_as))
os.remove(save_as)
self.assertFalse(os.path.exists(save_as))
# Also test if we can write it to the wiki
straxen.bokeh_to_wiki(fig)
straxen.bokeh_to_wiki(fig, save_as)
self.assertTrue(os.path.exists(save_as))
os.remove(save_as)
self.assertFalse(os.path.exists(save_as))
@unittest.skipIf(not straxen.utilix_is_configured(),
"No db access, cannot test!")
def test_nt_daq_plot(self):
"""Make an nt DAQ plot"""
self.st.daq_plot(
nt_test_run_id,
time_within=self.first_peak,
vmin=0.1,
vmax=1,
)
@unittest.skipIf(not straxen.utilix_is_configured(),
"No db access, cannot test!")
def test_nt_daq_plot_grouped(self):
"""Same as above grouped by ADC"""
self.st.plot_records_matrix(
nt_test_run_id,
time_within=self.first_peak,
vmin=0.1,
vmax=1,
group_by='ADC ID',
)
def test_records_matrix_downsample(self):
"""Test that downsampling works in the record matrix"""
self.st.records_matrix(nt_test_run_id,
time_within=self.first_event,
max_samples=20)
@unittest.skipIf(not straxen.utilix_is_configured(),
"No db access, cannot test!")
def test_load_corrected_positions(self):
"""Test that we can do st.load_corrected_positions"""
self.st.load_corrected_positions(nt_test_run_id,
time_within=self.first_peak)
@unittest.skipIf(not straxen.utilix_is_configured(),
"No db access, cannot test!")
def test_nv_event_display(self):
"""
Test NV event display for a single event.
"""
events_nv = self.st.get_array(nt_test_run_id, 'events_nv')
warning = ("Do 'rm ./strax_test_data' since your *_nv test "
"data in that folder is messing up this test.")
self.assertTrue(len(events_nv), warning)
self.st.make(nt_test_run_id, 'event_positions_nv')
self.st.plot_nveto_event_display(
nt_test_run_id,
time_within=events_nv[0],
)
with self.assertRaises(ValueError):
# If there is no data, we should raise a ValueError
self.st.plot_nveto_event_display(
nt_test_run_id,
time_range=[-1000, -900],
)
import strax
import straxen
import wfsim
from unittest import skipIf
@skipIf(not straxen.utilix_is_configured(), 'utilix is not configured')
def test_nt_context(register=None, context=None):
"""
Test a context if it is properly setup. To this end, we perform a
simple scan of the field "time" since all plugins should have this
field, if there is some option specified badly, we will quickly find
out since for this scan, we activate all the plugins.
:param register: Register a plugin (optional)
:param context: Test with some other context than the nT simulation
context
"""
if context is None:
context = straxen.contexts.xenonnt_simulation(
cmt_run_id_sim='010000', cmt_version='global_ONLINE')
assert isinstance(context, strax.Context), f'{context} is not a context'
if register is not None:
assert issubclass(register,
strax.Plugin), f'{register} is not a plugin'
# Search all plugins for the time field (each should have one)
context.search_field('time')
"""
Test certain interactions with the runsdatabase.
NB! this only works if one has access to the database. This does not
work e.g. on travis jobs and therefore the tests failing locally will
not show up in Pull Requests.
"""
import straxen
import os
import unittest
@unittest.skipIf(not straxen.utilix_is_configured(),
"No db access, cannot test!")
class TestSelectRuns(unittest.TestCase):
def test_select_runs(self, check_n_runs=2):
"""
Test (if we have a connection) if we can perform strax.select_runs
on the last two runs in the runs collection
:param check_n_runs: int, the number of runs we want to check
"""
self.assertTrue(check_n_runs >= 1)
st = straxen.contexts.xenonnt_online(use_rucio=False)
run_col = st.storage[0].collection
# Find the latest run in the runs collection
last_run = run_col.find_one(projection={
'number': 1
},
sort=[('number', -1)]).get('number')
class TestBasics(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
"""
For testing purposes, slightly alter the RucioFrontend such that
we can run tests outside of dali too
"""
if not straxen.utilix_is_configured():
return
if 'rcc' not in socket.getfqdn():
# If we are not on RCC, for testing, add some dummy site
straxen.RucioFrontend.local_rses = {
'UC_DALI_USERDISK': r'.rcc.',
'test_rucio': f'{socket.getfqdn()}'
}
straxen.RucioFrontend.get_rse_prefix = lambda *x: 'test_rucio'
# Some non-existing keys that we will try finding in the test cases.
cls.test_keys = [
strax.DataKey(run_id=run_id,
data_type='dtype',
lineage={
'dtype': ['Plugin', '0.0.0.', {}],
}) for run_id in ('-1', '-2')
]
@unittest.skipIf(not straxen.utilix_is_configured(),
"No db access, cannot test!")
def test_load_context_defaults(self):
"""Don't fail immediately if we start a context due to Rucio"""
st = straxen.contexts.xenonnt_online(
_minimum_run_number=10_000,
_maximum_run_number=10_010,
)
st.select_runs()
@unittest.skipIf(not straxen.utilix_is_configured(),
"No db access, cannot test!")
def test_find_local(self):
"""Make sure that we don't find the non existing data"""
rucio = straxen.RucioFrontend(include_remote=False, )
self.assertRaises(strax.DataNotAvailable, rucio.find,
self.test_keys[0])
@unittest.skipIf(not straxen.utilix_is_configured(),
"No db access, cannot test!")
def test_find_several_local(self):
"""Let's try finding some keys (won't be available)"""
rucio = straxen.RucioFrontend(include_remote=False, )
print(rucio)
found = rucio.find_several(self.test_keys)
# We shouldn't find any of these
assert found == [False for _ in self.test_keys]
@unittest.skipIf(not straxen.utilix_is_configured(),
"No db access, cannot test!")
def test_find_several_remote(self):
"""
Let's try running a find_several with the include remote.
This should fail but when no rucio is installed or else it
shouldn't find any data.
"""
try:
rucio = straxen.RucioFrontend(include_remote=True, )
except ImportError:
pass
else:
found = rucio.find_several(self.test_keys)
# We shouldn't find any of these
assert found == [False for _ in self.test_keys]
@unittest.skipIf(not straxen.utilix_is_configured(),
"No db access, cannot test!")
def test_find_local(self):
"""Make sure that we don't find the non existing data"""
run_db = straxen.RunDB(rucio_path='./rucio_test')
with self.assertRaises(strax.DataNotAvailable):
run_db.find(self.test_keys[0])
