def deldoc(self): """Deletes specified document in the database. """ mode, kwargs = self.mode, self.kwargs dbname = dbu.get_dbname(**kwargs) client = self.client() if not self.check_database(client, dbname): return detname = kwargs.get('detector', None) if detname is None: logger.warning( '%s needs in the collection name. Please specify the detector name.' % (mode)) colname = detname db, fs = dbu.db_and_fs(client, dbname) colnames = dbu.collection_names(db) if not (colname in colnames): # dbu.collection_exists(db, colname) logger.warning('db "%s" does not have collection "%s"' % (db.name, str(colname))) return col = dbu.collection(db, colname) logger.info('command mode: "%s" db: "%s" collection: "%s"' % (mode, db.name, str(colname))) defs = self.defs ctype = kwargs.get('ctype', None) run = kwargs.get('run', None) tsec = kwargs.get('time_sec', None) tstamp = kwargs.get('time_stamp', None) vers = kwargs.get('version', None) confirm = kwargs.get('confirm', False) query = {'detector': detname} if ctype != defs['ctype']: query['ctype'] = ctype if run != defs['run']: query['run'] = run if vers != defs['version']: query['version'] = vers #if tsec != defs['time_sec'] : query['time_sec'] = tsec if is_in_command_line('-s', '--time_sec'): query['time_sec'] = tsec if is_in_command_line('-t', '--time_stamp'): query['time_stamp'] = tstamp logger.info('query: %s' % str(query)) docs = dbu.find_docs(col, query) if docs is None or docs.count() == 0: logger.warning('Can not find document for query: %s' % str(query)) return for i, doc in enumerate(docs): msg = ' deldoc %2d:'%i + doc['time_stamp'] + ' ' + str(doc['time_sec'])\ + ' %s'%doc['ctype'].ljust(16) + ' %4d'%doc['run'] + ' ' + str(doc['id_data']) logger.info(msg) if confirm: dbu.delete_document_from_collection(col, doc['_id']) dbu.del_document_data(doc, fs) if not confirm: dbu.request_confirmation()
def delcol(self): """Deletes specified collection in the database. """ mode, kwargs = self.mode, self.kwargs dbname = dbu.get_dbname(**self.kwargs) client = self.client() if not self.check_database(client, dbname): return detname = kwargs.get('detector', None) if detname is None: logger.warning( '%s needs in the collection name. Please specify the detector name.' % (mode)) colname = detname db, fs = dbu.db_and_fs(client, dbname) colnames = dbu.collection_names(db) logger.info('"%s" deletes collection "%s" from database "%s"' % (mode, colname, db.name)) logger.info('Collections before "%s"' % str(colnames)) logger.info(dbu.database_fs_info(db, gap='')) if not (colname in colnames): logger.warning('db "%s" does not have collection "%s"' % (db.name, str(colname))) return if kwargs.get('confirm', False): col = dbu.collection(db, colname) dbu.del_collection_data( col, fs) # delete data in fs associated with collection col dbu.delete_collection_obj(col) logger.info('Collections after "%s"' % str(dbu.collection_names(db))) logger.info(dbu.database_fs_info(db, gap='')) else: dbu.request_confirmation()
def test_xtcav_calib_constants(fname= '/reg/d/psdm/XPP/xpptut15/calib/Xtcav::CalibV1/XrayTransportDiagnostic.0:Opal1000.0/pedestals/101-102.data',\ add_constants_to_db=False) : _, exp, _, cvers, detname, ctype, cfname = fname.rsplit('/',6) resp = parse_calib_file_name(cfname) begin, end, ext = resp if resp is not None else (None, None, None) det = detname_conversion(detname) run = begin dbname_exp = dbu.db_prefixed_name(exp) dbname_det = dbu.db_prefixed_name(det) print('LCLS1 Xtcav calibration file: %s' % fname) print('Parameters form path: exp:%s det:%s ctype:%s run:%s dbname_exp:%s dbname_det:%s'%\ (exp, det, ctype, run, dbname_exp, dbname_det)) #Save(ct,fname) o1 = Load(fname) d1 = dict_from_xtcav_calib_object(o1) print('Xtcav calibration constants as dict:\n', d1) if not add_constants_to_db : return #================================== #---- Delete databases for experiment and detector client = dbu.connect_to_server(HOST, PORT) print('Open client on host:%s port:%s' % (HOST, PORT)) print('Delete database %s'% dbname_exp) dbu.delete_database(client, dbname_exp) print('Delete database %s'% dbname_det) dbu.delete_database(client, dbname_det) #---- Add data to experiment and detector dbs print('Add Xtcav constants') kwargs = {'host' : HOST,\ 'port' : PORT,\ 'version' : 'V01',\ 'comment' : 'test of add-retrieve xtcav constants' } #insert_calib_data(data, *kwargs) dbu.insert_constants(o1, exp, det, ctype, run, time_sec='1000000000', **kwargs) #msg = dbu.database_info(client, dbname_exp, level=10) #print(msg) print('Xtcav constants inserted, now retrieve them from db:%s collection:%s' % (dbname_exp, det)) db, fs = dbu.db_and_fs(client, dbname_exp) col = dbu.collection(db, det) #for doc in col.find() : # print(doc) doc = dbu.find_doc(col, query={'ctype':ctype, 'run':run}) print('Found doc:\n', doc) o2 = dbu.get_data_for_doc(fs, doc) d2 = dict_from_xtcav_calib_object(o2) print('Xtcav calib object converted to dict:\n', d2) #print('cmp(d1,d2) :', str(d1==d2)) print('\nCompare dictionaries for Xtcav calib objects loaded directly from calib file and passed through the CDB') compare_dicts(d1,d2) client.close() return
def get(self): """Saves in file calibration constants from database. """ mode, kwargs = self.mode, self.kwargs defs = self.defs host = kwargs.get('host', None) port = kwargs.get('port', None) exp = kwargs.get('experiment', None) det = kwargs.get('detector', None) ctype = kwargs.get('ctype', None) run = kwargs.get('run', None) run_end = kwargs.get('run_end', None) tsec = kwargs.get('time_sec', None) tstamp = kwargs.get('time_stamp', None) vers = kwargs.get('version', None) fname = kwargs.get('iofname', None) query = {'detector': det, 'ctype': ctype} if run != defs['run']: query['run'] = {'$lte': run} query['run_end'] = {'$gte': run} if tsec != defs['time_sec']: query['time_sec'] = {'$lte': tsec} if vers != defs['version']: query['version'] = vers #logger.debug('query: %s' % str(query)) dbname = dbu.get_dbname(**kwargs) client = self.client() if not self.check_database(client, dbname): return detname = kwargs.get('detector', None) if detname is None: logger.warning( '%s needs in the collection name. Please specify the detector name.' % (mode)) colname = detname db, fs = dbu.db_and_fs(client, dbname) colnames = dbu.collection_names(db) if not (colname in colnames): # dbu.collection_exists(db, colname) logger.warning('db "%s" does not have collection "%s"' % (db.name, str(colname))) return col = dbu.collection(db, colname) logger.info('Search document in db "%s" collection "%s"' % (dbname, colname)) #client, expname, detname, db_exp, db_det, fs_exp, fs_det, col_exp, col_det =\ # dbu.connect(host=host, port=port, experiment=exp, detector=det, verbose=verb) #fs, doc = fs_exp, dbu.find_doc(col, query) #if doc is None : # fs, doc = fs_det, dbu.find_doc(col_det, query) # if doc is None : # logger.warning('Can not find document for query: %s' % str(query)) # return doc = dbu.find_doc(col, query) if doc is None: logger.warning('Can not find document for query: %s' % str(query)) return data = dbu.get_data_for_doc(fs, doc) if data is None: logger.warning('Can not load data for doc: %s' % str(doc)) return if fname is None: fname = 'clb-%s-%s-%s.npy' % (expname, det, ctype) if ctype == 'geometry': gu.save_textfile(data, fname, mode='w', verb=verb) elif os.path.splitext(fname)[1] == '.npy': np.save(fname, data, allow_pickle=False) else: save_txt(fname, data, fmt='%.2f') if verb: logger.info(info_ndarr(data, 'nda', first=0, last=3)) logger.info('Save constants in file: %s' % fname)
def get(self): """Gets constans from DB and saves them in file. """ mode, kwargs = self.mode, self.kwargs defs = self.defs host = kwargs.get('host', None) port = kwargs.get('port', None) exp = kwargs.get('experiment', None) det = kwargs.get('detector', None) ctype = kwargs.get('ctype', None) run = kwargs.get('run', None) run_end = kwargs.get('run_end', None) tsec = kwargs.get('time_sec', None) if is_in_command_line( '-s', '--time_sec') else None tstamp = kwargs.get('time_stamp', None) if is_in_command_line( '-t', '--time_stamp') else None vers = kwargs.get('version', None) prefix = kwargs.get('iofname', None) verb = self.loglevel == 'DEBUG' db_det, db_exp, colname, query = dbu.dbnames_collection_query( det, exp, ctype, run, tsec, vers) logger.debug('get: %s %s %s %s' % (db_det, db_exp, colname, str(query))) dbname = db_det if exp is None else db_exp client = self.client() if not self.check_database(client, dbname): return db, fs = dbu.db_and_fs(client, dbname) colnames = dbu.collection_names(db) if not (colname in colnames): # dbu.collection_exists(db, colname) logger.warning('db "%s" does not have collection "%s"' % (db.name, str(colname))) return col = dbu.collection(db, colname) logger.debug('Search document in db "%s" collection "%s"' % (dbname, colname)) doc = dbu.find_doc(col, query) if doc is None: logger.warning('Can not find document for query: %s' % str(query)) return logger.debug('get doc:', doc) data = dbu.get_data_for_doc(fs, doc) if data is None: logger.warning('Can not load data for doc: %s' % str(doc)) return if prefix is None: prefix = dbu.out_fname_prefix(**doc) dbu.save_doc_and_data_in_file(doc, data, prefix, control={ 'data': True, 'meta': True })