def get_mongo_url(hostname):
"""
Read url for mongo by reading the username and password from
straxen.get_secret.
:param hostname: The name of the host currently working on. If
this is an event-builder, we can use the gateway to
authenticate. Else we use either of the hosts in
default_mongo_url and backup_mongo_urls.
"""
if hostname.endswith('xenon.local'):
# So we are running strax on an event builder
username = straxen.get_secret('mongo_rdb_username')
password = straxen.get_secret('mongo_rdb_password')
url_base = 'gw:27017/admin'
mongo_url = f"mongodb://{username}:{password}@{url_base}"
else:
username = straxen.get_secret('rundb_username')
password = straxen.get_secret('rundb_password')
# try connection to the mongo database in this order
mongo_connections = [default_mongo_url, *backup_mongo_urls]
for url_base in mongo_connections:
try:
mongo_url = f"mongodb://{username}:{password}@{url_base}"
# Force server timeout if we cannot connect ot this url. If this
# does not raise an error, break and use this url
pymongo.MongoClient(mongo_url).server_info()
break
except pymongo.errors.ServerSelectionTimeoutError:
warn(f'Cannot connect to to Mongo url: {url_base}')
if url_base == mongo_connections[-1]:
raise pymongo.errors.ServerSelectionTimeoutError(
'Cannot connect to any Mongo url')
return mongo_url
def _update_context(st, max_workers, fallback_gains=None):
# Change config to allow for testing both multiprocessing and lazy mode
st.set_context_config({'forbid_creation_of': forbidden_plugins})
st.register(DummyRawRecords)
try:
straxen.get_secret('rundb_password')
# If you want to have quicker checks: always raise an ValueError as
# the CMT does take quite long to load the right corrections.
# ValueError
except ValueError:
# Okay so we cannot initize the runs-database. Let's just use some
# fallback values if they are specified.
if ('gain_model' in st.config
and st.config['gain_model'][0] == 'CMT_model'):
if fallback_gains is None:
# If you run into this error, go to the test_nT() - test and
# see for example how it is done there.
raise ValueError(
'Context uses CMT_model but no fallback_gains '
'are specified in test_plugins.py for this '
'context being tested')
else:
st.set_config({'gain_model': fallback_gains})
if max_workers - 1:
st.set_context_config({
'allow_multiprocess': True,
'allow_lazy': False,
'timeout': 60, # we don't want to build travis for ever
})
def test_secret():
"""
Check something in the sectets. This should not work because we
don't have any.
"""
try:
straxen.get_secret('somethingnonexistent')
except ValueError:
# Good we got some message we cannot load something that does
# not exist,
pass
def __init__(self, context=None):
"""
Interface to excess the XENONnT slow control data via python.
:param context: Context you are using e.g. st. This is needed
if you would like to query data via run_ids.
"""
try:
self.SCData_URL = straxen.get_secret('SCData_URL')
self.SCLastValue_URL = straxen.get_secret('SCLastValue_URL')
self.SCADA_SECRETS = straxen.get_secret('SCADA_SECRETS')
except ValueError:
raise ValueError('Cannot load SCADA information, from xenon'
' secrets. SCADAInterface cannot be used.')
self.context = context
def __init__(self, username='******', password=None, is_nt=True):
"""
:param username: corrections DB username
nt_analysis user has read only permission to corrections DB
cmt user has r/w permission to corrections DB and read permission to runsDB
:param password: DB password
:param is_nt: bool if True we are looking at nT if False we are looking at 1T
"""
self.username = username
self.is_nt = is_nt
if password is not None:
self.password = password
elif self.username.endswith('analysis'):
self.password = straxen.get_secret('rundb_password')
else:
raise ValueError(f'No password for {username}')
# Get the mongo url
runsdb_mongo_url = straxen.rundb.get_mongo_url(hostname=getfqdn())
_, _url = runsdb_mongo_url.split('@')
# Never use the admin authentication here.
_url = _url.replace('/admin', '/xenonnt')
self.interface = strax.CorrectionsInterface(
host=f'mongodb://{_url}',
username=self.username,
password=self.password,
database_name='corrections')
# Use the same client as the CorrectionsInterface
client = self.interface.client
if self.is_nt:
self.collection = client['xenonnt']['runs']
else:
self.collection = client['run']['runs_new']
def __init__(self,
mongo_url=None,
mongo_dbname=None,
mongo_collname=None,
runid_field='name',
s3_kwargs=None,
local_only=False,
new_data_path=None,
*args,
**kwargs):
"""
:param mongo_url: URL to Mongo runs database (including auth)
:param local_only: Do not show data as available if it would have to be
downloaded from a remote location.
:param new_data_path: Path where new files are to be written.
Defaults to None: do not write new data
New files will be registered in the runs db!
TODO: register under hostname alias (e.g. 'dali')
:param s3_kwargs: Arguments to initialize S3 backend (including auth)
:param runid_field: Rundb field to which strax's run_id concept
corresponds. Can be either
- 'name': values must be strings, for XENON1T
- 'number': values must be ints, for XENONnT DAQ tests
Other (kw)args are passed to StorageFrontend.__init__
TODO: disable S3 if secret keys not known
"""
super().__init__(*args, **kwargs)
self.local_only = local_only
self.new_data_path = new_data_path
if self.new_data_path is None:
self.readonly = True
self.runid_field = runid_field
if self.runid_field not in ['name', 'number']:
raise ValueError("Unrecognized runid_field option %s" %
self.runid_field)
if s3_kwargs is None:
s3_kwargs = dict(
aws_access_key_id=straxen.get_secret('s3_access_key_id'),
aws_secret_access_key=straxen.get_secret(
's3_secret_access_key'), # noqa
endpoint_url='http://ceph-s3.mwt2.org',
service_name='s3',
config=botocore.client.Config(connect_timeout=5,
retries=dict(max_attempts=10)))
if mongo_url is None:
mongo_url = default_mongo_url
self.client = pymongo.MongoClient(
mongo_url.format(username=straxen.get_secret('rundb_username'),
password=straxen.get_secret('rundb_password')))
if mongo_dbname is None:
mongo_dbname = default_mongo_dbname
if mongo_collname is None:
mongo_collname = default_mongo_collname
self.collection = self.client[mongo_dbname][mongo_collname]
self.backends = [
strax.S3Backend(**s3_kwargs),
strax.FileSytemBackend(),
]
# Construct mongo query for runs with available data.
# This depends on the machine you're running on.
self.hostname = socket.getfqdn()
self.available_query = [{'host': self.hostname}]
if not self.local_only:
self.available_query.append({'host': 'ceph-s3'})
# Go through known host aliases
for host_alias, regex in self.hosts.items():
if re.match(regex, self.hostname):
self.available_query.append({'host': host_alias})
