def __init__( self, readonly=True, file_database='files', config_identifier='config_name', collection=None, ): """ GridFsInterface :param readonly: bool, can one read or also write to the database. :param file_database: str, name of the database. Default should not be changed. :param config_identifier: str, header of the files that are saved in Gridfs :param collection: pymongo.collection.Collection, (Optional) PyMongo DataName Collection to bypass normal initiation using utilix. Should be an object of the form: pymongo.MongoClient(..).DATABASE_NAME.COLLECTION_NAME """ if collection is None: if not readonly: # We want admin access to start writing data! mongo_url = uconfig.get('rundb_admin', 'mongo_rdb_url') mongo_user = uconfig.get('rundb_admin', 'mongo_rdb_username') mongo_password = uconfig.get('rundb_admin', 'mongo_rdb_password') else: # We can safely use the Utilix defaults mongo_url = mongo_user = mongo_password = None # If no collection arg is passed, it defaults to the 'files' # collection, see for more details: # https://github.com/XENONnT/utilix/blob/master/utilix/rundb.py mongo_kwargs = { 'url': mongo_url, 'user': mongo_user, 'password': mongo_password, 'database': file_database, } # We can safely hard-code the collection as that is always # the same with GridFS. collection = utilix.rundb.pymongo_collection(**mongo_kwargs, collection='fs.files') else: # Check the user input is fine for what we want to do. if not isinstance(collection, pymongo_collection): raise ValueError('Provide PyMongo collection (see docstring)!') assert file_database is None, "Already provided a collection!" # Set collection and make sure it can at least do a 'find' operation self.collection = collection self.test_find() # This is the identifier under which we store the files. self.config_identifier = config_identifier # The GridFS used in this database self.grid_fs = gridfs.GridFS(collection.database)
def __init__(self, context=None, use_progress_bar=True): """ Interface to excess the XENONnT slow control data via python. :param context: Context you are using e.g. st. This is needed if you would like to query data via run_ids. :param use_progress_bar: Use a progress bar in the Scada interface """ try: self.SCData_URL = uconfig.get('scada', 'scdata_url') self.SCLastValue_URL = uconfig.get('scada', 'sclastvalue_url') self.SCADA_SECRETS = dict(QueryType=uconfig.get('scada', 'querytype'), username=uconfig.get('scada', 'username'), api_key=uconfig.get('scada', 'api_key') ) # Load parameters from the database. self.pmt_file = straxen.get_resource('PMTmap_SCADA.json', fmt='json') except ValueError as e: raise ValueError(f'Cannot load SCADA information, from your xenon' ' config. SCADAInterface cannot be used.') from e # Use a tqdm progress bar if requested. If a user does not want # a progress bar, just wrap it by a tuple self._use_progress_bar = use_progress_bar self.context = context
def get_mongo_uri(user_key='pymongo_user', pwd_key='pymongo_password', url_key='pymongo_url', header='RunDB'): user = uconfig.get(header, user_key) pwd = uconfig.get(header, pwd_key) url = uconfig.get(header, url_key) return f"mongodb://{user}:{pwd}@{url}"
def __init__(self, uri=None, take_only=None, database=None, col_name=default_online_collection, readonly=True, *args, **kwargs): if take_only is None: raise ValueError(f'Specify which data_types to accept! Otherwise ' f'the DataBase will be overloaded') if uri is None and readonly: uri = get_mongo_uri() elif uri is None and not readonly: # 'not readonly' means that you want to write. Let's get # your admin credentials: uri = get_mongo_uri(header='rundb_admin', user_key='mongo_rdb_username', pwd_key='mongo_rdb_password', url_key='mongo_rdb_url') if database is None: database = uconfig.get('RunDB', 'pymongo_database') super().__init__(uri=uri, database=database, take_only=take_only, col_name=col_name, *args, **kwargs) self.readonly = readonly
def __init__(self, context=None): """ Interface to excess the XENONnT slow control data via python. :param context: Context you are using e.g. st. This is needed if you would like to query data via run_ids. """ try: self.SCData_URL = uconfig.get('scada', 'scdata_url') self.SCLastValue_URL = uconfig.get('scada', 'sclastvalue_url') self.SCADA_SECRETS = dict( QueryType=uconfig.get('scada', 'querytype'), username=uconfig.get('scada', 'username'), api_key=uconfig.get('scada', 'api_key')) except ValueError as e: raise ValueError(f'Cannot load SCADA information, from your xenon' ' config. SCADAInterface cannot be used.') from e try: # Better to cache the file since is not large: with open(uconfig.get('scada', 'pmt_parameter_names')) as f: self.pmt_file = json.load(f) except (FileNotFoundError, ValueError): warnings.warn( ('Cannot load PMT parameter names from parameter file.' ' "find_pmt_names" is disabled for this session.')) self.pmt_file = None try: with open(uconfig.get('scada', 'parameter_readout_rate')) as f: self.read_out_rates = json.load(f) except (FileNotFoundError, ValueError) as e: raise FileNotFoundError( 'Cannot load file containing parameter sampling rates.') from e self.context = context
def __init__(self, minimum_run_number=7157, maximum_run_number=None, runid_field='name', local_only=False, new_data_path=None, reader_ini_name_is_mode=False, rucio_path=None, mongo_url=None, mongo_user=None, mongo_password=None, mongo_database=None, *args, **kwargs): """ :param minimum_run_number: Lowest number to consider :param maximum_run_number: Highest number to consider. When None (the default) consider all runs that are higher than the minimum_run_number. :param runid_field: Rundb field to which strax's run_id concept corresponds. Can be either - 'name': values must be strings, for XENON1T - 'number': values must be ints, for XENONnT DAQ tests :param local_only: Do not show data as available if it would have to be downloaded from a remote location. :param new_data_path: Path where new files are to be written. Defaults to None: do not write new data New files will be registered in the runs db! TODO: register under hostname alias (e.g. 'dali') :param reader_ini_name_is_mode: If True, will overwrite the 'mode' field with 'reader.ini.name'. :param rucio_path: What is the base path where Rucio is mounted :param mongo_url: URL to Mongo runs database (excl auth) :param mongo_user: user to Mongo runs database :param mongo_password: password to Mongo runs database :param mongo_database: database name of Mongo runs database Other (kw)args are passed to StorageFrontend.__init__ """ super().__init__(*args, **kwargs) self.local_only = local_only self.new_data_path = new_data_path self.reader_ini_name_is_mode = reader_ini_name_is_mode self.minimum_run_number = minimum_run_number self.maximum_run_number = maximum_run_number self.rucio_path = rucio_path if self.new_data_path is None: self.readonly = True self.runid_field = runid_field if self.runid_field not in ['name', 'number']: raise ValueError("Unrecognized runid_field option %s" % self.runid_field) self.hostname = socket.getfqdn() if not self.readonly and self.hostname.endswith('xenon.local'): # We want admin access to start writing data! mongo_url = uconfig.get('rundb_admin', 'mongo_rdb_url') mongo_user = uconfig.get('rundb_admin', 'mongo_rdb_username') mongo_password = uconfig.get('rundb_admin', 'mongo_rdb_password') mongo_database = uconfig.get('rundb_admin', 'mongo_rdb_database') # setup mongo kwargs... # utilix.rundb.pymongo_collection will take the following variables as kwargs # url: mongo url, including auth # user: the user # password: the password for the above user # database: the mongo database name # finally, it takes the collection name as an arg (not a kwarg). # if no collection arg is passed, it defaults to the runsDB collection # See github.com/XENONnT/utilix/blob/master/utilix/rundb.py for more details mongo_kwargs = { 'url': mongo_url, 'user': mongo_user, 'password': mongo_password, 'database': mongo_database } self.collection = utilix.rundb.xent_collection(**mongo_kwargs) # Do not delete the client! self.client = self.collection.database.client self.backends = [ strax.FileSytemBackend(), ] # Construct mongo query for runs with available data. # This depends on the machine you're running on. self.available_query = [{'host': self.hostname}] # Go through known host aliases for host_alias, regex in self.hosts.items(): if re.match(regex, self.hostname): self.available_query.append({'host': host_alias}) if self.rucio_path is not None: self.backends.append(strax.rucio(self.rucio_path)) # When querying for rucio, add that it should be dali-userdisk self.available_query.append({ 'host': 'rucio-catalogue', 'location': 'UC_DALI_USERDISK', 'status': 'transferred', })