def __init__(self, path='.', *args, **kwargs):
"""
:param path: Path to folder with data subfolders.
For other arguments, see DataRegistry base class.
"""
super().__init__(*args, **kwargs)
self.backends = [strax.FileSytemBackend()]
self.path = path
if not osp.exists(path):
os.makedirs(path)
def __init__(self, path='.', *args, deep_scan=False, **kwargs):
"""
:param path: Path to folder with data subfolders.
:param deep_scan: Let scan_runs scan over folders,
so even data for which no run-level metadata is available
is reported.
For other arguments, see DataRegistry base class.
"""
super().__init__(*args, **kwargs)
self.backends = [strax.FileSytemBackend()]
self.path = path
self.deep_scan = deep_scan
if not self.readonly and not osp.exists(self.path):
os.makedirs(self.path)
def __init__(self,
mongo_url=None,
mongo_dbname=None,
mongo_collname=None,
minimum_run_number=7157,
runid_field='name',
local_only=False,
new_data_path=None,
reader_ini_name_is_mode=False,
rucio_path=None,
*args,
**kwargs):
"""
:param mongo_url: URL to Mongo runs database (including auth)
:param local_only: Do not show data as available if it would have to be
downloaded from a remote location.
:param new_data_path: Path where new files are to be written.
Defaults to None: do not write new data
New files will be registered in the runs db!
TODO: register under hostname alias (e.g. 'dali')
:param runid_field: Rundb field to which strax's run_id concept
corresponds. Can be either
- 'name': values must be strings, for XENON1T
- 'number': values must be ints, for XENONnT DAQ tests
:param reader_ini_name_is_mode: If True, will overwrite the 'mode'
field with 'reader.ini.name'.
Other (kw)args are passed to StorageFrontend.__init__
"""
super().__init__(*args, **kwargs)
self.local_only = local_only
self.new_data_path = new_data_path
self.reader_ini_name_is_mode = reader_ini_name_is_mode
self.minimum_run_number = minimum_run_number
self.rucio_path = rucio_path
if self.new_data_path is None:
self.readonly = True
self.runid_field = runid_field
if self.runid_field not in ['name', 'number']:
raise ValueError("Unrecognized runid_field option %s" %
self.runid_field)
self.hostname = socket.getfqdn()
if mongo_url is None:
mongo_url = get_mongo_url(self.hostname)
self.client = pymongo.MongoClient(mongo_url)
if mongo_dbname is None:
mongo_dbname = default_mongo_dbname
if mongo_collname is None:
mongo_collname = default_mongo_collname
self.collection = self.client[mongo_dbname][mongo_collname]
self.backends = [
strax.FileSytemBackend(),
]
# Construct mongo query for runs with available data.
# This depends on the machine you're running on.
self.available_query = [{'host': self.hostname}]
# Go through known host aliases
for host_alias, regex in self.hosts.items():
if re.match(regex, self.hostname):
self.available_query.append({'host': host_alias})
if self.rucio_path is not None:
self.backends.append(strax.rucio(self.rucio_path))
# When querying for rucio, add that it should be dali-userdisk
self.available_query.append({
'host': 'rucio-catalogue',
'location': 'UC_DALI_USERDISK'
})
def __init__(self,
mongo_url=None,
mongo_dbname=None,
mongo_collname=None,
runid_field='name',
s3_kwargs=None,
local_only=False,
new_data_path=None,
reader_ini_name_is_mode=True,
*args,
**kwargs):
"""
:param mongo_url: URL to Mongo runs database (including auth)
:param local_only: Do not show data as available if it would have to be
downloaded from a remote location.
:param new_data_path: Path where new files are to be written.
Defaults to None: do not write new data
New files will be registered in the runs db!
TODO: register under hostname alias (e.g. 'dali')
:param s3_kwargs: Arguments to initialize S3 backend (including auth)
:param runid_field: Rundb field to which strax's run_id concept
corresponds. Can be either
- 'name': values must be strings, for XENON1T
- 'number': values must be ints, for XENONnT DAQ tests
:param reader_ini_name_is_mode: If True, will overwrite the 'mode'
field with 'reader.ini.name'.
Other (kw)args are passed to StorageFrontend.__init__
TODO: disable S3 if secret keys not known
"""
super().__init__(*args, **kwargs)
self.local_only = local_only
self.new_data_path = new_data_path
self.reader_ini_name_is_mode = reader_ini_name_is_mode
if self.new_data_path is None:
self.readonly = True
self.runid_field = runid_field
if self.runid_field not in ['name', 'number']:
raise ValueError("Unrecognized runid_field option %s" %
self.runid_field)
if s3_kwargs is None:
s3_kwargs = dict(
aws_access_key_id=nEXO_strax.get_secret('s3_access_key_id'),
aws_secret_access_key=nEXO_strax.get_secret(
's3_secret_access_key'), # noqa
endpoint_url='http://ceph-s3.mwt2.org',
service_name='s3',
config=botocore.client.Config(connect_timeout=5,
retries=dict(max_attempts=10)))
if mongo_url is None:
mongo_url = default_mongo_url
self.client = pymongo.MongoClient(
mongo_url.format(username=nEXO_strax.get_secret('rundb_username'),
password=nEXO_strax.get_secret('rundb_password')))
if mongo_dbname is None:
mongo_dbname = default_mongo_dbname
if mongo_collname is None:
mongo_collname = default_mongo_collname
self.collection = self.client[mongo_dbname][mongo_collname]
self.backends = [
strax.S3Backend(**s3_kwargs),
strax.FileSytemBackend(),
]
# Construct mongo query for runs with available data.
# This depends on the machine you're running on.
self.hostname = socket.getfqdn()
self.available_query = [{'host': self.hostname}]
if not self.local_only:
self.available_query.append({'host': 'ceph-s3'})
# Go through known host aliases
for host_alias, regex in self.hosts.items():
if re.match(regex, self.hostname):
self.available_query.append({'host': host_alias})
def __init__(self,
minimum_run_number=7157,
maximum_run_number=None,
runid_field='name',
local_only=False,
new_data_path=None,
reader_ini_name_is_mode=False,
rucio_path=None,
mongo_url=None,
mongo_user=None,
mongo_password=None,
mongo_database=None,
*args,
**kwargs):
"""
:param minimum_run_number: Lowest number to consider
:param maximum_run_number: Highest number to consider. When None
(the default) consider all runs that are higher than the
minimum_run_number.
:param runid_field: Rundb field to which strax's run_id concept
corresponds. Can be either
- 'name': values must be strings, for XENON1T
- 'number': values must be ints, for XENONnT DAQ tests
:param local_only: Do not show data as available if it would
have to be downloaded from a remote location.
:param new_data_path: Path where new files are to be written.
Defaults to None: do not write new data
New files will be registered in the runs db!
TODO: register under hostname alias (e.g. 'dali')
:param reader_ini_name_is_mode: If True, will overwrite the
'mode' field with 'reader.ini.name'.
:param rucio_path: What is the base path where Rucio is mounted
:param mongo_url: URL to Mongo runs database (excl auth)
:param mongo_user: user to Mongo runs database
:param mongo_password: password to Mongo runs database
:param mongo_database: database name of Mongo runs database
Other (kw)args are passed to StorageFrontend.__init__
"""
super().__init__(*args, **kwargs)
self.local_only = local_only
self.new_data_path = new_data_path
self.reader_ini_name_is_mode = reader_ini_name_is_mode
self.minimum_run_number = minimum_run_number
self.maximum_run_number = maximum_run_number
self.rucio_path = rucio_path
if self.new_data_path is None:
self.readonly = True
self.runid_field = runid_field
if self.runid_field not in ['name', 'number']:
raise ValueError("Unrecognized runid_field option %s" %
self.runid_field)
self.hostname = socket.getfqdn()
if not self.readonly and self.hostname.endswith('xenon.local'):
# We want admin access to start writing data!
mongo_url = uconfig.get('rundb_admin', 'mongo_rdb_url')
mongo_user = uconfig.get('rundb_admin', 'mongo_rdb_username')
mongo_password = uconfig.get('rundb_admin', 'mongo_rdb_password')
mongo_database = uconfig.get('rundb_admin', 'mongo_rdb_database')
# setup mongo kwargs...
# utilix.rundb.pymongo_collection will take the following variables as kwargs
# url: mongo url, including auth
# user: the user
# password: the password for the above user
# database: the mongo database name
# finally, it takes the collection name as an arg (not a kwarg).
# if no collection arg is passed, it defaults to the runsDB collection
# See github.com/XENONnT/utilix/blob/master/utilix/rundb.py for more details
mongo_kwargs = {
'url': mongo_url,
'user': mongo_user,
'password': mongo_password,
'database': mongo_database
}
self.collection = utilix.rundb.xent_collection(**mongo_kwargs)
# Do not delete the client!
self.client = self.collection.database.client
self.backends = [
strax.FileSytemBackend(),
]
# Construct mongo query for runs with available data.
# This depends on the machine you're running on.
self.available_query = [{'host': self.hostname}]
# Go through known host aliases
for host_alias, regex in self.hosts.items():
if re.match(regex, self.hostname):
self.available_query.append({'host': host_alias})
if self.rucio_path is not None:
self.backends.append(strax.rucio(self.rucio_path))
# When querying for rucio, add that it should be dali-userdisk
self.available_query.append({
'host': 'rucio-catalogue',
'location': 'UC_DALI_USERDISK',
'status': 'transferred',
})
