def info_webclient(**kwargs):
width = kwargs.get('width', 24)
ptrn = mu.db_prefixed_name('') if kwargs.get('cdbonly', False) else None
dbnames = database_names(url=cc.URL, pattern=ptrn)
dbname = mu.get_dbname(**kwargs)
if dbname is None:
s = '\n=== web client %s contains %d databases for name pattern "%s":\n%s\n\n' % (cc.URL, len(dbnames), str(ptrn), str_formatted_list(dbnames))
for name in dbnames:
colnames = collection_names(name, url=cc.URL)
s += '%s %2d cols: %s\n' % (str(name).ljust(width), len(colnames), str(colnames))
return s
if not (dbname in dbnames):
return '\n=== database %s is not found in the list of known:\n%s' % (dbname, str_formatted_list(dbnames))
colname = mu.get_colname(**kwargs)
colnames = collection_names(dbname, url=cc.URL)
if colname is None:
if colnames is None: return '\n=== colnames is None: database %s is empty ???' % (dbname)
s = '\n=== database %s contains %d collections: %s\n' % (dbname, len(colnames), str(colnames))
for cname in colnames:
s += '%s\n' % info_docs(dbname, cname)
return s
if not(colname in colnames):
return '\n=== database %s does not have collection %s in the list: %s' % (dbname, colname, str(colnames))
docid = kwargs.get('docid', None)
if docid is None: return info_docs(dbname, colname)
return info_doc(dbname, colname, docid)
def importdb(self):
"""Imports database. Equivalent to: mongorestore -d <dbname> --archive <filename>
mongorestore --archive cdb-2018-03-09T10-19-30-cdb-cxix25115.arc --db calib-cxi12345
"""
host, port, dbname, fname = self.host_port_dbname_fname()
dbu.importdb(host, port, dbname, fname)
def print_content(self):
dbname = dbu.get_dbname(**self.kwargs)
client = self.client()
if dbname is not None:
logger.info(dbu.database_info(client, dbname, level=3))
else:
logger.info(dbu.client_info(client, level=2))
def collection_names(db):
"""
"""
if isinstance(db, str):
client = connect_client()
db = dbu.database(client, db)
return dbu.collection_names(db)
def add_data_and_two_docs(data, exp, det, url=cc.URL_KRB, krbheaders=cc.KRBHEADERS, **kwargs):
""" Adds data and document to experiment and detector data bases.
"""
t0_sec = time()
detname = pro_detector_name(det)
colname = detname
dbname_exp = mu.db_prefixed_name(exp)
dbname_det = mu.db_prefixed_name(detname)
id_data_exp = add_data(dbname_exp, data, url, krbheaders)
id_data_det = add_data(dbname_det, data, url, krbheaders)
if None in (id_data_exp, id_data_det): return None
doc = mu.docdic(data, id_data_exp, **kwargs)
logger.debug(mu.doc_info(doc, fmt=' %s:%s')) #sep='\n %16s : %s'
id_doc_exp = add_document(dbname_exp, colname, doc, url, krbheaders)
doc['id_data'] = id_data_det # override
doc['id_exp'] = id_doc_exp # add
id_doc_det = add_document(dbname_det, colname, doc, url, krbheaders)
if None in (id_doc_exp, id_doc_det): return None
msg = 'Add 2 data and docs time %.6f sec' % (time()-t0_sec)\
+ '\n - data in %s/gridfs id: %s and doc in collection %s id: %s' % (dbname_exp, id_data_exp, colname, id_doc_exp)\
+ '\n - data in %s/gridfs id: %s and doc in collection %s id: %s' % (dbname_det, id_data_det, colname, id_doc_det)
logger.debug(msg)
return id_data_exp, id_data_det, id_doc_exp, id_doc_det
def add_data_and_doc(data,
_dbname,
_colname,
url=cc.URL_KRB,
krbheaders=cc.KRBHEADERS,
**kwargs):
"""Adds data and document to the db
"""
logger.debug('add_data_and_doc kwargs: %s' % str(kwargs))
# check permission
t0_sec = time()
if not valid_post_privilege(_dbname, url_krb=url): return None
id_data = add_data(_dbname, data, url, krbheaders)
if id_data is None: return None
doc = mu.docdic(data, id_data, **kwargs) # ObjectId(id_data)???
logger.debug(mu.doc_info(doc, fmt=' %s:%s')) #sep='\n %16s : %s'
id_doc = add_document(_dbname, _colname, doc, url, krbheaders)
if id_doc is None: return None
msg = 'Add data and doc time %.6f sec' % (time()-t0_sec)\
+ '\n - data in %s/gridfs id: %s and doc in collection %s id: %s' % (_dbname, id_data, _colname, id_doc)
logger.debug(msg)
return id_data, id_doc
def scan_calib_for_experiment(exp='cxix25615', **kwargs):
host = kwargs.get('host', None)
port = kwargs.get('port', None)
user = kwargs.get('user', None)
upwd = kwargs.get('upwd', None)
verbose = kwargs.get('verbose', False)
client = dbu.connect_to_server(host, port, user, upwd)
dbname = dbu.db_prefixed_name(exp)
if dbu.database_exists(client, dbname):
msg = 'Experiment %s already has a database. Consider to delete it from the list:\n%s'%\
(exp, str(dbu.database_names(client)))+\
'\nBefore adding consider to delete existing DB using command: cdb deldb --dbname %s -C -u <username> -p <password>' % dbname
logger.warning(msg)
return
dircalib = nm.dir_calib(exp)
#if verbose :
logger.info('Scan: %s' % dircalib)
for dir0 in gu.get_list_of_files_in_dir_for_part_fname(dircalib,
pattern='::'):
if not os.path.isdir(dir0): continue
calibvers = os.path.basename(dir0)
logger.debug(' %s ' % calibvers)
for dir1 in gu.get_list_of_files_in_dir_for_part_fname(dir0,
pattern=':'):
if not os.path.isdir(dir1): continue
detname = os.path.basename(dir1)
detname_m = detname_conversion(detname)
logger.debug(' %s' % detname_m)
for cftype in gu.get_list_of_files_in_dir(dir1):
if not (cftype in cc.list_calib_names): continue
dir2 = '%s/%s' % (dir1, cftype)
if not os.path.isdir(dir2): continue
logger.debug(' %s' % cftype)
cfdir = '%s/%s/%s/%s' % (dircalib, calibvers, detname, cftype)
listdicts = history_list_of_dicts('%s/HISTORY' % cfdir,
verbose)
#logger.debug('XXX listdicts %s' % listdicts)
count = 0
for fname in gu.get_list_of_files_in_dir(dir2):
logger.debug(' %s' % fname)
if fname == 'HISTORY': continue
if os.path.splitext(fname)[1] != '.data': continue
logger.debug(' XXX begin adding: %s %s %s %s' %
(dircalib, detname_m, cftype, fname))
add_calib_file_to_cdb(exp, dircalib, calibvers, detname_m,
cftype, fname, cfdir, listdicts,
**kwargs)
count += 1
logger.info(' converted %3d files from: %s' % (count, cfdir))
def add_calib_file_to_cdb(exp, dircalib, calibvers, detname, cftype, fname,
cfdir, listdicts, **kwargs):
"""
"""
d = history_dict_for_file(listdicts, fname)
resp = parse_calib_file_name(fname)
begin, end, ext = resp if resp is not None else (None, None, None)
if begin is not None: begin = int(begin)
if None in (begin, end, ext): return
fpath = '%s/%s' % (cfdir, fname)
verbose = kwargs.get('verbose', False)
data = gu.load_textfile(fpath, verbose) if cftype in ('geometry','code_geometry') else\
load_xtcav_calib_file(fpath) if is_xtcav(calibvers, cftype) else\
load_txt(fpath) # using NDArrIO
if isinstance(data, dict):
serialize_dict(data)
logger.debug(info_dict(data))
#print_dict(data)
#data = json.dumps(data) # (data,ensure_ascii=True) json.dumps converts dict -> str # .replace("'", '"') for json
data = str(data)
if isinstance(data, np.ndarray):
check_data_shape(data, detname, cftype)
begin_time, end_time = run_begin_end_time(exp, int(begin))
if verbose:
ndu.print_ndarr(data, 'scan calib: data')
msg = 'scan calib: %s %s %s %s %s %s %s %s %s' % (
exp, cfdir, fname, begin, end, ext, calibvers, detname, cftype)
logger.info(msg)
logger.info('begin_time: %s end_time: %s' % (begin_time, end_time))
if data is None:
msg = 'data is None, conversion is dropped for for file: %s' % fpath
logger.warning(msg)
return
kwargs['run'] = begin
kwargs['run_end'] = end
kwargs['detector'] = detname
kwargs['ctype'] = cftype
kwargs['time_sec'] = begin_time
kwargs['end_time'] = end_time
kwargs['time_stamp'] = dbu._timestamp(begin_time)
kwargs['extpars'] = d if d is not None else {
} # just in case save entire history dict
#kwargs['comment'] = 'HISTORY: %s' % d.get('comment', '')
dbu.insert_calib_data(data, **kwargs)
def exportdb(self):
"""Exports database. Equivalent to: mongodump -d <dbname> -o <filename>
mongodump --host psanaphi105 --port 27017 --db calib-cxi12345 --archive=db.20180122.arc
"""
host, port, dbname, fname = self.host_port_dbname_fname()
tstamp = gu.str_tstamp(fmt='%Y-%m-%dT%H-%M-%S')
fname = 'cdb-%s-%s.arc' % (tstamp, dbname) if fname is None else fname
dbu.exportdb(host, port, dbname, fname)
def add(self):
"""Adds calibration constants to database from file.
"""
kwa = self.kwargs
fname = kwa.get('iofname', 'None')
ctype = kwa.get('ctype', 'None')
dtype = kwa.get('dtype', 'None')
verb = self.loglevel == 'DEBUG'
data = mu.data_from_file(fname, ctype, dtype, verb)
mu.insert_calib_data(data, **kwa)
def delete_collections(dic_db_cols):
"""Delete collections specified in the dic_db_cols consisting of pairs {dbname:lstcols}
"""
msg = 'Delete collections:'
client = connect_client()
for dbname, lstcols in dic_db_cols.items():
db = dbu.database(client, dbname)
msg += '\nFrom database: %s delete collections:\n %s' % (
dbname, '\n '.join(lstcols))
dbu.delete_collections(db, lstcols)
logger.debug(msg)
def add(self):
"""Adds calibration constants to database from file.
"""
kwargs = self.kwargs
fname = kwargs.get('iofname', 'None')
ctype = kwargs.get('ctype', 'None')
assert os.path.exists(fname), 'File "%s" DOES NOT EXIST' % fname
ext = os.path.splitext(fname)[-1]
data = gu.load_textfile(fname, verb=False) if ctype == 'geometry' else\
np.load(fname) if ext == '.npy' else\
load_txt(fname)
dbu.insert_calib_data(data, **kwargs)
def check_database(self, client, dbname):
if dbu.database_exists(client, dbname):
return True
logger.warning(
'Database "%s" is not available. See for deteals: cdb print' %
(dbname))
return False
def client(self):
kwargs = self.kwargs
host = kwargs.get('host', None)
port = kwargs.get('port', None)
msg = 'MongoDB client host:%s port:%d' % (host, port)
logger.info(msg)
return dbu.connect_to_server(host, port)
def calib_constants(det,
exp=None,
ctype='pedestals',
run=None,
time_sec=None,
vers=None,
url=cc.URL):
"""Returns calibration constants and document with metadata for specified parameters.
To get meaningful constants, at least a few parameters must be specified, e.g.:
- det, ctype, time_sec
- det, ctype, version
- det, exp, ctype, run
- det, exp, ctype, time_sec
- det, exp, ctype, run, version
etc...
"""
db_det, db_exp, colname, query = mu.dbnames_collection_query(
det, exp, ctype, run, time_sec, vers)
logger.debug('get_constants: %s %s %s %s' %
(db_det, db_exp, colname, str(query)))
dbname = db_det if exp is None else db_exp
doc = find_doc(dbname, colname, query, url)
if doc is None:
# commented out by cpo since this happens routinely the way
# that Mona is fetching calibration constants in psana.
#logger.warning('document is not available for query: %s' % str(query))
return (None, None)
return (get_data_for_doc(dbname, colname, doc, url), doc)
def _add_detector_name(dbname, colname, detname, detnum):
""" Adds document for detector names and returns short detector name for long input name detname.
"""
check_kerberos_ticket()
doc = mu._doc_detector_name(detname, colname, detnum)
id_doc = add_document(dbname, colname, doc) #, url, krbheaders)
return doc.get('short', None) if id_doc is not None else None
def calib_constants_all_types(det,
exp=None,
run=None,
time_sec=None,
vers=None,
url=cc.URL):
""" returns constants for all ctype-s
"""
ctype = None
db_det, db_exp, colname, query = mu.dbnames_collection_query(
det, exp, ctype, run, time_sec, vers)
dbname = db_det if exp is None else db_exp
docs = find_docs(dbname, colname, query, url)
#logger.debug('find_docs: number of docs found: %d' % len(docs))
if docs is None: return None
ctypes = set([d.get('ctype', None) for d in docs])
ctypes.discard(None)
logger.debug('calib_constants_all_types - found ctypes: %s' % str(ctypes))
resp = {}
for ct in ctypes:
docs_for_type = [d for d in docs if d.get('ctype', None) == ct]
doc = select_latest_doc(docs_for_type, query)
if doc is None: continue
resp[ct] = (get_data_for_doc(dbname, colname, doc, url), doc)
return resp
def test_add_data_and_two_docs(exp=TEST_EXPNAME, det=TEST_DETNAME):
from psana.pyalgos.generic.Utils import get_login
t0_sec = time()
kwa = {'user' : get_login(),
'experiment': exp,
'detector' : det,
'ctype' : 'testnda',
'run' : 123,
'time_sec' : t0_sec,
'time_stamp': mu._timestamp(int(t0_sec)),
}
data = mu.get_test_nda()
id_data_exp, id_data_det, id_doc_exp, id_doc_det =\
add_data_and_two_docs(data, exp, det, url=cc.URL_KRB, krbheaders=cc.KRBHEADERS, **kwa)
print('time to insert data and two docs: %.6f sec' % (time()-t0_sec))
def connect_client(host=None,
port=None,
user=cp.user,
upwd=cp.upwd): # user=dbu.cc.USERNAME
_host = cp.cdb_host.value() if host is None else host
_port = cp.cdb_port.value() if port is None else port
#logger.debug('CMDBBUtils: Connect client to host: %s port: %d user: %s upwd: %s' % (_host, _port, user, upwd))
return dbu.connect_to_server(_host, _port, user, upwd)
def test_insert_constants(expname=TEST_EXPNAME, detname=TEST_DETNAME, ctype='test_ctype', runnum=10, data='test text sampele'):
""" Inserts constants using direct MongoDB interface from MDBUtils.
"""
import psana.pyalgos.generic.Utils as gu
print('test_delete_database 1:', database_names())
#txt = '%s\nThis is a string\n to test\ncalibration storage' % gu.str_tstamp()
#data, ctype = txt, 'testtext'; logger.debug('txt: %s' % str(data))
#data, ctype = get_test_nda(), 'testnda'; logger.debug(info_ndarr(data, 'nda'))
#data, ctype = get_test_dic(), 'testdict'; logger.debug('dict: %s' % str(data))
kwa = {'user' : gu.get_login()}
t0_sec = time()
ts = gu.str_tstamp(fmt='%Y-%m-%dT%H:%M:%S%z', time_sec=t0_sec)
mu.insert_constants('%s - saved at %s'%(data,ts), expname, detname, ctype, runnum+int(tname), int(t0_sec),\
time_stamp=ts, **kwa)
print('test_delete_database 2:', database_names())
def client(self):
kwargs = self.kwargs
host = kwargs.get('host', None)
port = kwargs.get('port', None)
user = kwargs.get('user', None)
upwd = kwargs.get('upwd', None)
ctout = kwargs.get('ctout', 5000)
stout = kwargs.get('stout', 30000)
return dbu.connect_to_server(host, port, user, upwd, ctout, stout)
def add_data_and_doc(data, dbname, colname, url=cc.URL_KRB, krbheaders=cc.KRBHEADERS, **kwargs):
"""Adds data and document to the db
"""
id_data = add_data(dbname, data, url, krbheaders)
if id_data is None : return None
doc = mu.docdic(data, id_data, **kwargs) # ObjectId(id_data)???
id_doc = add_document(dbname, colname, doc, url, krbheaders)
if id_doc is None : return None
return id_data, id_doc
def add_data_and_two_docs(data,
exp,
det,
url=cc.URL_KRB,
krbheaders=cc.KRBHEADERS,
**kwargs):
""" Check permission and add data and document to experiment and detector data bases.
"""
logger.debug('add_data_and_two_docs kwargs: %s' % str(kwargs))
detname = pro_detector_name(det, add_shortname=True)
colname = detname
dbname_exp = mu.db_prefixed_name(exp)
dbname_det = mu.db_prefixed_name(detname)
kwargs['detector'] = detname # ex: epix10ka_000001
kwargs['shortname'] = detname # ex: epix10ka_000001
kwargs['longname'] = det # ex: epix10ka_<_uniqueid>
#kwargs['detname'] = det_name # already in kwargs ex: epixquad
resp = add_data_and_doc(data,
dbname_exp,
colname,
url=url,
krbheaders=krbheaders,
**kwargs)
if resp is None: return None
id_data_exp, id_doc_exp = resp
#logger.warning('Deployment of constants in the %s needs in expert-privilage\n' % dbname_det)
#if not kwargs.get('confirm', False):
#mu.request_confirmation()
#return id_data_exp, None, id_doc_exp, None
kwargs['id_data_exp'] = id_data_exp # override
kwargs['id_doc_exp'] = id_doc_exp # add
resp = add_data_and_doc(data,
dbname_det,
colname,
url=url,
krbheaders=krbheaders,
**kwargs)
id_data_det, id_doc_det = resp if resp is not None else (None, None)
return id_data_exp, id_data_det, id_doc_exp, id_doc_det
def test_get_random_doc_and_data_ids(det='cspad_0001'):
dbname = mu.db_prefixed_name(det)
colname = det
doc = find_doc(dbname, colname, query={'ctype':'pedestals'})
print('Pick up any doc for dbname:%s colname:%s pedestals: ' % (dbname,colname))
print('Document: %s' % str(doc))
id_doc = doc.get('_id', None)
id_data = doc.get('id_data', None)
print('_id : %s id_data : %s' % (id_doc, id_data))
return id_doc, id_data, dbname, colname
def dbnames_collection_query(det, exp=None, ctype='pedestals', run=None, time_sec=None, vers=None, dtype=None, dbsuffix=''):
"""wrapper for MDBUtils.dbnames_collection_query,
- which should receive short detector name, othervice uses direct interface to DB
"""
short = pro_detector_name(det)
logger.debug('short: %s dbsuffix: %s' % (short, dbsuffix))
resp = list(mu.dbnames_collection_query(short, exp, ctype, run, time_sec, vers, dtype))
if dbsuffix: resp[0] = detector_dbname(short, dbsuffix=dbsuffix)
return resp
def add(self):
"""Adds calibration constants to database from file.
"""
kwargs = self.kwargs
fname = kwargs.get('iofname', 'None')
ctype = kwargs.get('ctype', 'None')
dtype = kwargs.get('dtype', 'None')
verb = self.loglevel == 'DEBUG'
assert os.path.exists(fname), 'File "%s" DOES NOT EXIST' % fname
ext = os.path.splitext(fname)[-1]
data = gu.load_textfile(fname, verb=verb) if ctype == 'geometry' or dtype in ('str', 'txt', 'text') else\
load_xtcav_calib_file(fname) if dtype == 'xtcav' else\
np.load(fname) if ext == '.npy' else\
gu.load_json(fname) if ext == '.json' or dtype == 'json' else\
gu.load_pickle(fname) if ext == '.pkl' or dtype in ('pkl', 'pickle') else\
load_txt(fname) # input NDArrIO
dbu.insert_calib_data(data, **kwargs)
def add_calib_file_to_cdb(exp, dircalib, calibvers, detname, cftype, fname, cfdir, listdicts, **kwargs) :
"""
"""
d = history_dict_for_file(listdicts, fname)
resp = parse_calib_file_name(fname)
begin, end, ext = resp if resp is not None else (None, None, None)
if None in (begin, end, ext) : return
fpath = '%s/%s' % (cfdir, fname)
verbose = kwargs.get('verbose', False)
data = gu.load_textfile(fpath, verbose) if cftype in ('geometry','code_geometry') else\
load_xtcav_calib_file(fpath) if is_xtcav(calibvers, cftype) else\
load_txt(fpath) # using NDArrIO
begin_time, end_time = run_begin_end_time(exp, int(begin))
if verbose :
ndu.print_ndarr(data, 'scan calib: data')
msg = 'scan calib: %s %s %s %s %s %s %s %s %s' % (exp, cfdir, fname, begin, end, ext, calibvers, detname, cftype)
logger.info(msg)
logger.info('begin_time: %s end_time: %s' % (begin_time, end_time))
if data is None :
msg = 'data is None, conversion is dropped for for file: %s' % fpath
logger.warning(msg)
return
kwargs['run'] = begin
kwargs['run_end'] = end
kwargs['detector'] = detname
kwargs['ctype'] = cftype
kwargs['time_sec'] = begin_time
kwargs['end_time'] = end_time
kwargs['time_stamp'] = dbu._timestamp(begin_time)
kwargs['extpars'] = d # just in case save entire history dict
#kwargs['comment'] = 'HISTORY: %s' % d.get('comment', '')
dbu.insert_calib_data(data, **kwargs)
def add_data(dbname, data, url=cc.URL_KRB, krbheaders=cc.KRBHEADERS):
"""Adds binary data to the database/gridfs.
"""
import io
headers = dict(krbheaders) # krbheaders <class 'dict'>
headers['Content-Type'] = 'application/octet-stream'
f = io.BytesIO(mu.encode_data(data)) # io.StringIO(data)
d = f.read()
logger.debug('add_data byte-data:', d)
resp = post(url + dbname + '/gridfs/', headers=headers, data=d)
logger.debug('add_data: to %s/gridfs/ resp: %s' % (dbname, resp.text))
return resp.json().get('_id', None)
def collection_info(dbname, cname, **kwa):
"""Returns (str) info about collection documents.
"""
s = 'DB %s collection %s' % (dbname, cname)
docs = find_docs(dbname, cname)
if not docs: return s
s += ' contains %d docs\n' % len(docs)
docs = sorted(docs, key=my_sort_parameter) #, reverse=True
doc = docs[0]
s += '\n %s' % mu.document_keys(doc) # str(doc.keys())
_, title = mu.document_info(doc, **kwa)
s += '\n doc# %s' % title
for idoc, doc in enumerate(docs):
vals,_ = mu.document_info(doc, **kwa)
s += '\n %4d %s' % (idoc, vals)
return s
def get_data_for_doc(dbname, colname, doc, url=cc.URL):
"""Returns data from GridFS using doc.
"""
logger.debug('get_data_for_doc: %s', str(doc))
idd = doc.get('id_data', None)
if idd is None :
logger.debug("get_data_for_doc: key 'id_data' is missing in selected document...")
return None
r2 = request('%s/%s/gridfs/%s'%(url,dbname,idd))
s = r2.content
return mu.object_from_data_string(s, doc)