def info_webclient(**kwargs): width = kwargs.get('width', 24) ptrn = mu.db_prefixed_name('') if kwargs.get('cdbonly', False) else None dbnames = database_names(url=cc.URL, pattern=ptrn) dbname = mu.get_dbname(**kwargs) if dbname is None: s = '\n=== web client %s contains %d databases for name pattern "%s":\n%s\n\n' % (cc.URL, len(dbnames), str(ptrn), str_formatted_list(dbnames)) for name in dbnames: colnames = collection_names(name, url=cc.URL) s += '%s %2d cols: %s\n' % (str(name).ljust(width), len(colnames), str(colnames)) return s if not (dbname in dbnames): return '\n=== database %s is not found in the list of known:\n%s' % (dbname, str_formatted_list(dbnames)) colname = mu.get_colname(**kwargs) colnames = collection_names(dbname, url=cc.URL) if colname is None: if colnames is None: return '\n=== colnames is None: database %s is empty ???' % (dbname) s = '\n=== database %s contains %d collections: %s\n' % (dbname, len(colnames), str(colnames)) for cname in colnames: s += '%s\n' % info_docs(dbname, cname) return s if not(colname in colnames): return '\n=== database %s does not have collection %s in the list: %s' % (dbname, colname, str(colnames)) docid = kwargs.get('docid', None) if docid is None: return info_docs(dbname, colname) return info_doc(dbname, colname, docid)
def importdb(self): """Imports database. Equivalent to: mongorestore -d <dbname> --archive <filename> mongorestore --archive cdb-2018-03-09T10-19-30-cdb-cxix25115.arc --db calib-cxi12345 """ host, port, dbname, fname = self.host_port_dbname_fname() dbu.importdb(host, port, dbname, fname)
def print_content(self): dbname = dbu.get_dbname(**self.kwargs) client = self.client() if dbname is not None: logger.info(dbu.database_info(client, dbname, level=3)) else: logger.info(dbu.client_info(client, level=2))
def collection_names(db): """ """ if isinstance(db, str): client = connect_client() db = dbu.database(client, db) return dbu.collection_names(db)
def add_data_and_two_docs(data, exp, det, url=cc.URL_KRB, krbheaders=cc.KRBHEADERS, **kwargs): """ Adds data and document to experiment and detector data bases. """ t0_sec = time() detname = pro_detector_name(det) colname = detname dbname_exp = mu.db_prefixed_name(exp) dbname_det = mu.db_prefixed_name(detname) id_data_exp = add_data(dbname_exp, data, url, krbheaders) id_data_det = add_data(dbname_det, data, url, krbheaders) if None in (id_data_exp, id_data_det): return None doc = mu.docdic(data, id_data_exp, **kwargs) logger.debug(mu.doc_info(doc, fmt=' %s:%s')) #sep='\n %16s : %s' id_doc_exp = add_document(dbname_exp, colname, doc, url, krbheaders) doc['id_data'] = id_data_det # override doc['id_exp'] = id_doc_exp # add id_doc_det = add_document(dbname_det, colname, doc, url, krbheaders) if None in (id_doc_exp, id_doc_det): return None msg = 'Add 2 data and docs time %.6f sec' % (time()-t0_sec)\ + '\n - data in %s/gridfs id: %s and doc in collection %s id: %s' % (dbname_exp, id_data_exp, colname, id_doc_exp)\ + '\n - data in %s/gridfs id: %s and doc in collection %s id: %s' % (dbname_det, id_data_det, colname, id_doc_det) logger.debug(msg) return id_data_exp, id_data_det, id_doc_exp, id_doc_det
def add_data_and_doc(data, _dbname, _colname, url=cc.URL_KRB, krbheaders=cc.KRBHEADERS, **kwargs): """Adds data and document to the db """ logger.debug('add_data_and_doc kwargs: %s' % str(kwargs)) # check permission t0_sec = time() if not valid_post_privilege(_dbname, url_krb=url): return None id_data = add_data(_dbname, data, url, krbheaders) if id_data is None: return None doc = mu.docdic(data, id_data, **kwargs) # ObjectId(id_data)??? logger.debug(mu.doc_info(doc, fmt=' %s:%s')) #sep='\n %16s : %s' id_doc = add_document(_dbname, _colname, doc, url, krbheaders) if id_doc is None: return None msg = 'Add data and doc time %.6f sec' % (time()-t0_sec)\ + '\n - data in %s/gridfs id: %s and doc in collection %s id: %s' % (_dbname, id_data, _colname, id_doc) logger.debug(msg) return id_data, id_doc
def scan_calib_for_experiment(exp='cxix25615', **kwargs): host = kwargs.get('host', None) port = kwargs.get('port', None) user = kwargs.get('user', None) upwd = kwargs.get('upwd', None) verbose = kwargs.get('verbose', False) client = dbu.connect_to_server(host, port, user, upwd) dbname = dbu.db_prefixed_name(exp) if dbu.database_exists(client, dbname): msg = 'Experiment %s already has a database. Consider to delete it from the list:\n%s'%\ (exp, str(dbu.database_names(client)))+\ '\nBefore adding consider to delete existing DB using command: cdb deldb --dbname %s -C -u <username> -p <password>' % dbname logger.warning(msg) return dircalib = nm.dir_calib(exp) #if verbose : logger.info('Scan: %s' % dircalib) for dir0 in gu.get_list_of_files_in_dir_for_part_fname(dircalib, pattern='::'): if not os.path.isdir(dir0): continue calibvers = os.path.basename(dir0) logger.debug(' %s ' % calibvers) for dir1 in gu.get_list_of_files_in_dir_for_part_fname(dir0, pattern=':'): if not os.path.isdir(dir1): continue detname = os.path.basename(dir1) detname_m = detname_conversion(detname) logger.debug(' %s' % detname_m) for cftype in gu.get_list_of_files_in_dir(dir1): if not (cftype in cc.list_calib_names): continue dir2 = '%s/%s' % (dir1, cftype) if not os.path.isdir(dir2): continue logger.debug(' %s' % cftype) cfdir = '%s/%s/%s/%s' % (dircalib, calibvers, detname, cftype) listdicts = history_list_of_dicts('%s/HISTORY' % cfdir, verbose) #logger.debug('XXX listdicts %s' % listdicts) count = 0 for fname in gu.get_list_of_files_in_dir(dir2): logger.debug(' %s' % fname) if fname == 'HISTORY': continue if os.path.splitext(fname)[1] != '.data': continue logger.debug(' XXX begin adding: %s %s %s %s' % (dircalib, detname_m, cftype, fname)) add_calib_file_to_cdb(exp, dircalib, calibvers, detname_m, cftype, fname, cfdir, listdicts, **kwargs) count += 1 logger.info(' converted %3d files from: %s' % (count, cfdir))
def add_calib_file_to_cdb(exp, dircalib, calibvers, detname, cftype, fname, cfdir, listdicts, **kwargs): """ """ d = history_dict_for_file(listdicts, fname) resp = parse_calib_file_name(fname) begin, end, ext = resp if resp is not None else (None, None, None) if begin is not None: begin = int(begin) if None in (begin, end, ext): return fpath = '%s/%s' % (cfdir, fname) verbose = kwargs.get('verbose', False) data = gu.load_textfile(fpath, verbose) if cftype in ('geometry','code_geometry') else\ load_xtcav_calib_file(fpath) if is_xtcav(calibvers, cftype) else\ load_txt(fpath) # using NDArrIO if isinstance(data, dict): serialize_dict(data) logger.debug(info_dict(data)) #print_dict(data) #data = json.dumps(data) # (data,ensure_ascii=True) json.dumps converts dict -> str # .replace("'", '"') for json data = str(data) if isinstance(data, np.ndarray): check_data_shape(data, detname, cftype) begin_time, end_time = run_begin_end_time(exp, int(begin)) if verbose: ndu.print_ndarr(data, 'scan calib: data') msg = 'scan calib: %s %s %s %s %s %s %s %s %s' % ( exp, cfdir, fname, begin, end, ext, calibvers, detname, cftype) logger.info(msg) logger.info('begin_time: %s end_time: %s' % (begin_time, end_time)) if data is None: msg = 'data is None, conversion is dropped for for file: %s' % fpath logger.warning(msg) return kwargs['run'] = begin kwargs['run_end'] = end kwargs['detector'] = detname kwargs['ctype'] = cftype kwargs['time_sec'] = begin_time kwargs['end_time'] = end_time kwargs['time_stamp'] = dbu._timestamp(begin_time) kwargs['extpars'] = d if d is not None else { } # just in case save entire history dict #kwargs['comment'] = 'HISTORY: %s' % d.get('comment', '') dbu.insert_calib_data(data, **kwargs)
def exportdb(self): """Exports database. Equivalent to: mongodump -d <dbname> -o <filename> mongodump --host psanaphi105 --port 27017 --db calib-cxi12345 --archive=db.20180122.arc """ host, port, dbname, fname = self.host_port_dbname_fname() tstamp = gu.str_tstamp(fmt='%Y-%m-%dT%H-%M-%S') fname = 'cdb-%s-%s.arc' % (tstamp, dbname) if fname is None else fname dbu.exportdb(host, port, dbname, fname)
def add(self): """Adds calibration constants to database from file. """ kwa = self.kwargs fname = kwa.get('iofname', 'None') ctype = kwa.get('ctype', 'None') dtype = kwa.get('dtype', 'None') verb = self.loglevel == 'DEBUG' data = mu.data_from_file(fname, ctype, dtype, verb) mu.insert_calib_data(data, **kwa)
def delete_collections(dic_db_cols): """Delete collections specified in the dic_db_cols consisting of pairs {dbname:lstcols} """ msg = 'Delete collections:' client = connect_client() for dbname, lstcols in dic_db_cols.items(): db = dbu.database(client, dbname) msg += '\nFrom database: %s delete collections:\n %s' % ( dbname, '\n '.join(lstcols)) dbu.delete_collections(db, lstcols) logger.debug(msg)
def add(self): """Adds calibration constants to database from file. """ kwargs = self.kwargs fname = kwargs.get('iofname', 'None') ctype = kwargs.get('ctype', 'None') assert os.path.exists(fname), 'File "%s" DOES NOT EXIST' % fname ext = os.path.splitext(fname)[-1] data = gu.load_textfile(fname, verb=False) if ctype == 'geometry' else\ np.load(fname) if ext == '.npy' else\ load_txt(fname) dbu.insert_calib_data(data, **kwargs)
def check_database(self, client, dbname): if dbu.database_exists(client, dbname): return True logger.warning( 'Database "%s" is not available. See for deteals: cdb print' % (dbname)) return False
def client(self): kwargs = self.kwargs host = kwargs.get('host', None) port = kwargs.get('port', None) msg = 'MongoDB client host:%s port:%d' % (host, port) logger.info(msg) return dbu.connect_to_server(host, port)
def calib_constants(det, exp=None, ctype='pedestals', run=None, time_sec=None, vers=None, url=cc.URL): """Returns calibration constants and document with metadata for specified parameters. To get meaningful constants, at least a few parameters must be specified, e.g.: - det, ctype, time_sec - det, ctype, version - det, exp, ctype, run - det, exp, ctype, time_sec - det, exp, ctype, run, version etc... """ db_det, db_exp, colname, query = mu.dbnames_collection_query( det, exp, ctype, run, time_sec, vers) logger.debug('get_constants: %s %s %s %s' % (db_det, db_exp, colname, str(query))) dbname = db_det if exp is None else db_exp doc = find_doc(dbname, colname, query, url) if doc is None: # commented out by cpo since this happens routinely the way # that Mona is fetching calibration constants in psana. #logger.warning('document is not available for query: %s' % str(query)) return (None, None) return (get_data_for_doc(dbname, colname, doc, url), doc)
def _add_detector_name(dbname, colname, detname, detnum): """ Adds document for detector names and returns short detector name for long input name detname. """ check_kerberos_ticket() doc = mu._doc_detector_name(detname, colname, detnum) id_doc = add_document(dbname, colname, doc) #, url, krbheaders) return doc.get('short', None) if id_doc is not None else None
def calib_constants_all_types(det, exp=None, run=None, time_sec=None, vers=None, url=cc.URL): """ returns constants for all ctype-s """ ctype = None db_det, db_exp, colname, query = mu.dbnames_collection_query( det, exp, ctype, run, time_sec, vers) dbname = db_det if exp is None else db_exp docs = find_docs(dbname, colname, query, url) #logger.debug('find_docs: number of docs found: %d' % len(docs)) if docs is None: return None ctypes = set([d.get('ctype', None) for d in docs]) ctypes.discard(None) logger.debug('calib_constants_all_types - found ctypes: %s' % str(ctypes)) resp = {} for ct in ctypes: docs_for_type = [d for d in docs if d.get('ctype', None) == ct] doc = select_latest_doc(docs_for_type, query) if doc is None: continue resp[ct] = (get_data_for_doc(dbname, colname, doc, url), doc) return resp
def test_add_data_and_two_docs(exp=TEST_EXPNAME, det=TEST_DETNAME): from psana.pyalgos.generic.Utils import get_login t0_sec = time() kwa = {'user' : get_login(), 'experiment': exp, 'detector' : det, 'ctype' : 'testnda', 'run' : 123, 'time_sec' : t0_sec, 'time_stamp': mu._timestamp(int(t0_sec)), } data = mu.get_test_nda() id_data_exp, id_data_det, id_doc_exp, id_doc_det =\ add_data_and_two_docs(data, exp, det, url=cc.URL_KRB, krbheaders=cc.KRBHEADERS, **kwa) print('time to insert data and two docs: %.6f sec' % (time()-t0_sec))
def connect_client(host=None, port=None, user=cp.user, upwd=cp.upwd): # user=dbu.cc.USERNAME _host = cp.cdb_host.value() if host is None else host _port = cp.cdb_port.value() if port is None else port #logger.debug('CMDBBUtils: Connect client to host: %s port: %d user: %s upwd: %s' % (_host, _port, user, upwd)) return dbu.connect_to_server(_host, _port, user, upwd)
def test_insert_constants(expname=TEST_EXPNAME, detname=TEST_DETNAME, ctype='test_ctype', runnum=10, data='test text sampele'): """ Inserts constants using direct MongoDB interface from MDBUtils. """ import psana.pyalgos.generic.Utils as gu print('test_delete_database 1:', database_names()) #txt = '%s\nThis is a string\n to test\ncalibration storage' % gu.str_tstamp() #data, ctype = txt, 'testtext'; logger.debug('txt: %s' % str(data)) #data, ctype = get_test_nda(), 'testnda'; logger.debug(info_ndarr(data, 'nda')) #data, ctype = get_test_dic(), 'testdict'; logger.debug('dict: %s' % str(data)) kwa = {'user' : gu.get_login()} t0_sec = time() ts = gu.str_tstamp(fmt='%Y-%m-%dT%H:%M:%S%z', time_sec=t0_sec) mu.insert_constants('%s - saved at %s'%(data,ts), expname, detname, ctype, runnum+int(tname), int(t0_sec),\ time_stamp=ts, **kwa) print('test_delete_database 2:', database_names())
def client(self): kwargs = self.kwargs host = kwargs.get('host', None) port = kwargs.get('port', None) user = kwargs.get('user', None) upwd = kwargs.get('upwd', None) ctout = kwargs.get('ctout', 5000) stout = kwargs.get('stout', 30000) return dbu.connect_to_server(host, port, user, upwd, ctout, stout)
def add_data_and_doc(data, dbname, colname, url=cc.URL_KRB, krbheaders=cc.KRBHEADERS, **kwargs): """Adds data and document to the db """ id_data = add_data(dbname, data, url, krbheaders) if id_data is None : return None doc = mu.docdic(data, id_data, **kwargs) # ObjectId(id_data)??? id_doc = add_document(dbname, colname, doc, url, krbheaders) if id_doc is None : return None return id_data, id_doc
def add_data_and_two_docs(data, exp, det, url=cc.URL_KRB, krbheaders=cc.KRBHEADERS, **kwargs): """ Check permission and add data and document to experiment and detector data bases. """ logger.debug('add_data_and_two_docs kwargs: %s' % str(kwargs)) detname = pro_detector_name(det, add_shortname=True) colname = detname dbname_exp = mu.db_prefixed_name(exp) dbname_det = mu.db_prefixed_name(detname) kwargs['detector'] = detname # ex: epix10ka_000001 kwargs['shortname'] = detname # ex: epix10ka_000001 kwargs['longname'] = det # ex: epix10ka_<_uniqueid> #kwargs['detname'] = det_name # already in kwargs ex: epixquad resp = add_data_and_doc(data, dbname_exp, colname, url=url, krbheaders=krbheaders, **kwargs) if resp is None: return None id_data_exp, id_doc_exp = resp #logger.warning('Deployment of constants in the %s needs in expert-privilage\n' % dbname_det) #if not kwargs.get('confirm', False): #mu.request_confirmation() #return id_data_exp, None, id_doc_exp, None kwargs['id_data_exp'] = id_data_exp # override kwargs['id_doc_exp'] = id_doc_exp # add resp = add_data_and_doc(data, dbname_det, colname, url=url, krbheaders=krbheaders, **kwargs) id_data_det, id_doc_det = resp if resp is not None else (None, None) return id_data_exp, id_data_det, id_doc_exp, id_doc_det
def test_get_random_doc_and_data_ids(det='cspad_0001'): dbname = mu.db_prefixed_name(det) colname = det doc = find_doc(dbname, colname, query={'ctype':'pedestals'}) print('Pick up any doc for dbname:%s colname:%s pedestals: ' % (dbname,colname)) print('Document: %s' % str(doc)) id_doc = doc.get('_id', None) id_data = doc.get('id_data', None) print('_id : %s id_data : %s' % (id_doc, id_data)) return id_doc, id_data, dbname, colname
def dbnames_collection_query(det, exp=None, ctype='pedestals', run=None, time_sec=None, vers=None, dtype=None, dbsuffix=''): """wrapper for MDBUtils.dbnames_collection_query, - which should receive short detector name, othervice uses direct interface to DB """ short = pro_detector_name(det) logger.debug('short: %s dbsuffix: %s' % (short, dbsuffix)) resp = list(mu.dbnames_collection_query(short, exp, ctype, run, time_sec, vers, dtype)) if dbsuffix: resp[0] = detector_dbname(short, dbsuffix=dbsuffix) return resp
def add(self): """Adds calibration constants to database from file. """ kwargs = self.kwargs fname = kwargs.get('iofname', 'None') ctype = kwargs.get('ctype', 'None') dtype = kwargs.get('dtype', 'None') verb = self.loglevel == 'DEBUG' assert os.path.exists(fname), 'File "%s" DOES NOT EXIST' % fname ext = os.path.splitext(fname)[-1] data = gu.load_textfile(fname, verb=verb) if ctype == 'geometry' or dtype in ('str', 'txt', 'text') else\ load_xtcav_calib_file(fname) if dtype == 'xtcav' else\ np.load(fname) if ext == '.npy' else\ gu.load_json(fname) if ext == '.json' or dtype == 'json' else\ gu.load_pickle(fname) if ext == '.pkl' or dtype in ('pkl', 'pickle') else\ load_txt(fname) # input NDArrIO dbu.insert_calib_data(data, **kwargs)
def add_calib_file_to_cdb(exp, dircalib, calibvers, detname, cftype, fname, cfdir, listdicts, **kwargs) : """ """ d = history_dict_for_file(listdicts, fname) resp = parse_calib_file_name(fname) begin, end, ext = resp if resp is not None else (None, None, None) if None in (begin, end, ext) : return fpath = '%s/%s' % (cfdir, fname) verbose = kwargs.get('verbose', False) data = gu.load_textfile(fpath, verbose) if cftype in ('geometry','code_geometry') else\ load_xtcav_calib_file(fpath) if is_xtcav(calibvers, cftype) else\ load_txt(fpath) # using NDArrIO begin_time, end_time = run_begin_end_time(exp, int(begin)) if verbose : ndu.print_ndarr(data, 'scan calib: data') msg = 'scan calib: %s %s %s %s %s %s %s %s %s' % (exp, cfdir, fname, begin, end, ext, calibvers, detname, cftype) logger.info(msg) logger.info('begin_time: %s end_time: %s' % (begin_time, end_time)) if data is None : msg = 'data is None, conversion is dropped for for file: %s' % fpath logger.warning(msg) return kwargs['run'] = begin kwargs['run_end'] = end kwargs['detector'] = detname kwargs['ctype'] = cftype kwargs['time_sec'] = begin_time kwargs['end_time'] = end_time kwargs['time_stamp'] = dbu._timestamp(begin_time) kwargs['extpars'] = d # just in case save entire history dict #kwargs['comment'] = 'HISTORY: %s' % d.get('comment', '') dbu.insert_calib_data(data, **kwargs)
def add_data(dbname, data, url=cc.URL_KRB, krbheaders=cc.KRBHEADERS): """Adds binary data to the database/gridfs. """ import io headers = dict(krbheaders) # krbheaders <class 'dict'> headers['Content-Type'] = 'application/octet-stream' f = io.BytesIO(mu.encode_data(data)) # io.StringIO(data) d = f.read() logger.debug('add_data byte-data:', d) resp = post(url + dbname + '/gridfs/', headers=headers, data=d) logger.debug('add_data: to %s/gridfs/ resp: %s' % (dbname, resp.text)) return resp.json().get('_id', None)
def collection_info(dbname, cname, **kwa): """Returns (str) info about collection documents. """ s = 'DB %s collection %s' % (dbname, cname) docs = find_docs(dbname, cname) if not docs: return s s += ' contains %d docs\n' % len(docs) docs = sorted(docs, key=my_sort_parameter) #, reverse=True doc = docs[0] s += '\n %s' % mu.document_keys(doc) # str(doc.keys()) _, title = mu.document_info(doc, **kwa) s += '\n doc# %s' % title for idoc, doc in enumerate(docs): vals,_ = mu.document_info(doc, **kwa) s += '\n %4d %s' % (idoc, vals) return s
def get_data_for_doc(dbname, colname, doc, url=cc.URL): """Returns data from GridFS using doc. """ logger.debug('get_data_for_doc: %s', str(doc)) idd = doc.get('id_data', None) if idd is None : logger.debug("get_data_for_doc: key 'id_data' is missing in selected document...") return None r2 = request('%s/%s/gridfs/%s'%(url,dbname,idd)) s = r2.content return mu.object_from_data_string(s, doc)