def __init__(self, path='.', *args, **kwargs): """ :param path: Path to folder with data subfolders. For other arguments, see DataRegistry base class. """ super().__init__(*args, **kwargs) self.backends = [strax.FileSytemBackend()] self.path = path if not osp.exists(path): os.makedirs(path)
def __init__(self, path='.', *args, deep_scan=False, **kwargs): """ :param path: Path to folder with data subfolders. :param deep_scan: Let scan_runs scan over folders, so even data for which no run-level metadata is available is reported. For other arguments, see DataRegistry base class. """ super().__init__(*args, **kwargs) self.backends = [strax.FileSytemBackend()] self.path = path self.deep_scan = deep_scan if not self.readonly and not osp.exists(self.path): os.makedirs(self.path)
def __init__(self, mongo_url=None, mongo_dbname=None, mongo_collname=None, minimum_run_number=7157, runid_field='name', local_only=False, new_data_path=None, reader_ini_name_is_mode=False, rucio_path=None, *args, **kwargs): """ :param mongo_url: URL to Mongo runs database (including auth) :param local_only: Do not show data as available if it would have to be downloaded from a remote location. :param new_data_path: Path where new files are to be written. Defaults to None: do not write new data New files will be registered in the runs db! TODO: register under hostname alias (e.g. 'dali') :param runid_field: Rundb field to which strax's run_id concept corresponds. Can be either - 'name': values must be strings, for XENON1T - 'number': values must be ints, for XENONnT DAQ tests :param reader_ini_name_is_mode: If True, will overwrite the 'mode' field with 'reader.ini.name'. Other (kw)args are passed to StorageFrontend.__init__ """ super().__init__(*args, **kwargs) self.local_only = local_only self.new_data_path = new_data_path self.reader_ini_name_is_mode = reader_ini_name_is_mode self.minimum_run_number = minimum_run_number self.rucio_path = rucio_path if self.new_data_path is None: self.readonly = True self.runid_field = runid_field if self.runid_field not in ['name', 'number']: raise ValueError("Unrecognized runid_field option %s" % self.runid_field) self.hostname = socket.getfqdn() if mongo_url is None: mongo_url = get_mongo_url(self.hostname) self.client = pymongo.MongoClient(mongo_url) if mongo_dbname is None: mongo_dbname = default_mongo_dbname if mongo_collname is None: mongo_collname = default_mongo_collname self.collection = self.client[mongo_dbname][mongo_collname] self.backends = [ strax.FileSytemBackend(), ] # Construct mongo query for runs with available data. # This depends on the machine you're running on. self.available_query = [{'host': self.hostname}] # Go through known host aliases for host_alias, regex in self.hosts.items(): if re.match(regex, self.hostname): self.available_query.append({'host': host_alias}) if self.rucio_path is not None: self.backends.append(strax.rucio(self.rucio_path)) # When querying for rucio, add that it should be dali-userdisk self.available_query.append({ 'host': 'rucio-catalogue', 'location': 'UC_DALI_USERDISK' })
def __init__(self, mongo_url=None, mongo_dbname=None, mongo_collname=None, runid_field='name', s3_kwargs=None, local_only=False, new_data_path=None, reader_ini_name_is_mode=True, *args, **kwargs): """ :param mongo_url: URL to Mongo runs database (including auth) :param local_only: Do not show data as available if it would have to be downloaded from a remote location. :param new_data_path: Path where new files are to be written. Defaults to None: do not write new data New files will be registered in the runs db! TODO: register under hostname alias (e.g. 'dali') :param s3_kwargs: Arguments to initialize S3 backend (including auth) :param runid_field: Rundb field to which strax's run_id concept corresponds. Can be either - 'name': values must be strings, for XENON1T - 'number': values must be ints, for XENONnT DAQ tests :param reader_ini_name_is_mode: If True, will overwrite the 'mode' field with 'reader.ini.name'. Other (kw)args are passed to StorageFrontend.__init__ TODO: disable S3 if secret keys not known """ super().__init__(*args, **kwargs) self.local_only = local_only self.new_data_path = new_data_path self.reader_ini_name_is_mode = reader_ini_name_is_mode if self.new_data_path is None: self.readonly = True self.runid_field = runid_field if self.runid_field not in ['name', 'number']: raise ValueError("Unrecognized runid_field option %s" % self.runid_field) if s3_kwargs is None: s3_kwargs = dict( aws_access_key_id=nEXO_strax.get_secret('s3_access_key_id'), aws_secret_access_key=nEXO_strax.get_secret( 's3_secret_access_key'), # noqa endpoint_url='http://ceph-s3.mwt2.org', service_name='s3', config=botocore.client.Config(connect_timeout=5, retries=dict(max_attempts=10))) if mongo_url is None: mongo_url = default_mongo_url self.client = pymongo.MongoClient( mongo_url.format(username=nEXO_strax.get_secret('rundb_username'), password=nEXO_strax.get_secret('rundb_password'))) if mongo_dbname is None: mongo_dbname = default_mongo_dbname if mongo_collname is None: mongo_collname = default_mongo_collname self.collection = self.client[mongo_dbname][mongo_collname] self.backends = [ strax.S3Backend(**s3_kwargs), strax.FileSytemBackend(), ] # Construct mongo query for runs with available data. # This depends on the machine you're running on. self.hostname = socket.getfqdn() self.available_query = [{'host': self.hostname}] if not self.local_only: self.available_query.append({'host': 'ceph-s3'}) # Go through known host aliases for host_alias, regex in self.hosts.items(): if re.match(regex, self.hostname): self.available_query.append({'host': host_alias})
def __init__(self, minimum_run_number=7157, maximum_run_number=None, runid_field='name', local_only=False, new_data_path=None, reader_ini_name_is_mode=False, rucio_path=None, mongo_url=None, mongo_user=None, mongo_password=None, mongo_database=None, *args, **kwargs): """ :param minimum_run_number: Lowest number to consider :param maximum_run_number: Highest number to consider. When None (the default) consider all runs that are higher than the minimum_run_number. :param runid_field: Rundb field to which strax's run_id concept corresponds. Can be either - 'name': values must be strings, for XENON1T - 'number': values must be ints, for XENONnT DAQ tests :param local_only: Do not show data as available if it would have to be downloaded from a remote location. :param new_data_path: Path where new files are to be written. Defaults to None: do not write new data New files will be registered in the runs db! TODO: register under hostname alias (e.g. 'dali') :param reader_ini_name_is_mode: If True, will overwrite the 'mode' field with 'reader.ini.name'. :param rucio_path: What is the base path where Rucio is mounted :param mongo_url: URL to Mongo runs database (excl auth) :param mongo_user: user to Mongo runs database :param mongo_password: password to Mongo runs database :param mongo_database: database name of Mongo runs database Other (kw)args are passed to StorageFrontend.__init__ """ super().__init__(*args, **kwargs) self.local_only = local_only self.new_data_path = new_data_path self.reader_ini_name_is_mode = reader_ini_name_is_mode self.minimum_run_number = minimum_run_number self.maximum_run_number = maximum_run_number self.rucio_path = rucio_path if self.new_data_path is None: self.readonly = True self.runid_field = runid_field if self.runid_field not in ['name', 'number']: raise ValueError("Unrecognized runid_field option %s" % self.runid_field) self.hostname = socket.getfqdn() if not self.readonly and self.hostname.endswith('xenon.local'): # We want admin access to start writing data! mongo_url = uconfig.get('rundb_admin', 'mongo_rdb_url') mongo_user = uconfig.get('rundb_admin', 'mongo_rdb_username') mongo_password = uconfig.get('rundb_admin', 'mongo_rdb_password') mongo_database = uconfig.get('rundb_admin', 'mongo_rdb_database') # setup mongo kwargs... # utilix.rundb.pymongo_collection will take the following variables as kwargs # url: mongo url, including auth # user: the user # password: the password for the above user # database: the mongo database name # finally, it takes the collection name as an arg (not a kwarg). # if no collection arg is passed, it defaults to the runsDB collection # See github.com/XENONnT/utilix/blob/master/utilix/rundb.py for more details mongo_kwargs = { 'url': mongo_url, 'user': mongo_user, 'password': mongo_password, 'database': mongo_database } self.collection = utilix.rundb.xent_collection(**mongo_kwargs) # Do not delete the client! self.client = self.collection.database.client self.backends = [ strax.FileSytemBackend(), ] # Construct mongo query for runs with available data. # This depends on the machine you're running on. self.available_query = [{'host': self.hostname}] # Go through known host aliases for host_alias, regex in self.hosts.items(): if re.match(regex, self.hostname): self.available_query.append({'host': host_alias}) if self.rucio_path is not None: self.backends.append(strax.rucio(self.rucio_path)) # When querying for rucio, add that it should be dali-userdisk self.available_query.append({ 'host': 'rucio-catalogue', 'location': 'UC_DALI_USERDISK', 'status': 'transferred', })