def add_data_and_two_docs(data, exp, det, url=cc.URL_KRB, krbheaders=cc.KRBHEADERS, **kwargs):
""" Adds data and document to experiment and detector data bases.
"""
t0_sec = time()
detname = pro_detector_name(det)
colname = detname
dbname_exp = mu.db_prefixed_name(exp)
dbname_det = mu.db_prefixed_name(detname)
id_data_exp = add_data(dbname_exp, data, url, krbheaders)
id_data_det = add_data(dbname_det, data, url, krbheaders)
if None in (id_data_exp, id_data_det): return None
doc = mu.docdic(data, id_data_exp, **kwargs)
logger.debug(mu.doc_info(doc, fmt=' %s:%s')) #sep='\n %16s : %s'
id_doc_exp = add_document(dbname_exp, colname, doc, url, krbheaders)
doc['id_data'] = id_data_det # override
doc['id_exp'] = id_doc_exp # add
id_doc_det = add_document(dbname_det, colname, doc, url, krbheaders)
if None in (id_doc_exp, id_doc_det): return None
msg = 'Add 2 data and docs time %.6f sec' % (time()-t0_sec)\
+ '\n - data in %s/gridfs id: %s and doc in collection %s id: %s' % (dbname_exp, id_data_exp, colname, id_doc_exp)\
+ '\n - data in %s/gridfs id: %s and doc in collection %s id: %s' % (dbname_det, id_data_det, colname, id_doc_det)
logger.debug(msg)
return id_data_exp, id_data_det, id_doc_exp, id_doc_det
def info_webclient(**kwargs):
width = kwargs.get('width', 24)
ptrn = mu.db_prefixed_name('') if kwargs.get('cdbonly', False) else None
dbnames = database_names(url=cc.URL, pattern=ptrn)
dbname = mu.get_dbname(**kwargs)
if dbname is None:
s = '\n=== web client %s contains %d databases for name pattern "%s":\n%s\n\n' % (cc.URL, len(dbnames), str(ptrn), str_formatted_list(dbnames))
for name in dbnames:
colnames = collection_names(name, url=cc.URL)
s += '%s %2d cols: %s\n' % (str(name).ljust(width), len(colnames), str(colnames))
return s
if not (dbname in dbnames):
return '\n=== database %s is not found in the list of known:\n%s' % (dbname, str_formatted_list(dbnames))
colname = mu.get_colname(**kwargs)
colnames = collection_names(dbname, url=cc.URL)
if colname is None:
if colnames is None: return '\n=== colnames is None: database %s is empty ???' % (dbname)
s = '\n=== database %s contains %d collections: %s\n' % (dbname, len(colnames), str(colnames))
for cname in colnames:
s += '%s\n' % info_docs(dbname, cname)
return s
if not(colname in colnames):
return '\n=== database %s does not have collection %s in the list: %s' % (dbname, colname, str(colnames))
docid = kwargs.get('docid', None)
if docid is None: return info_docs(dbname, colname)
return info_doc(dbname, colname, docid)
def scan_calib_for_experiment(exp='cxix25615', **kwargs):
host = kwargs.get('host', None)
port = kwargs.get('port', None)
user = kwargs.get('user', None)
upwd = kwargs.get('upwd', None)
verbose = kwargs.get('verbose', False)
client = dbu.connect_to_server(host, port, user, upwd)
dbname = dbu.db_prefixed_name(exp)
if dbu.database_exists(client, dbname):
msg = 'Experiment %s already has a database. Consider to delete it from the list:\n%s'%\
(exp, str(dbu.database_names(client)))+\
'\nBefore adding consider to delete existing DB using command: cdb deldb --dbname %s -C -u <username> -p <password>' % dbname
logger.warning(msg)
return
dircalib = nm.dir_calib(exp)
#if verbose :
logger.info('Scan: %s' % dircalib)
for dir0 in gu.get_list_of_files_in_dir_for_part_fname(dircalib,
pattern='::'):
if not os.path.isdir(dir0): continue
calibvers = os.path.basename(dir0)
logger.debug(' %s ' % calibvers)
for dir1 in gu.get_list_of_files_in_dir_for_part_fname(dir0,
pattern=':'):
if not os.path.isdir(dir1): continue
detname = os.path.basename(dir1)
detname_m = detname_conversion(detname)
logger.debug(' %s' % detname_m)
for cftype in gu.get_list_of_files_in_dir(dir1):
if not (cftype in cc.list_calib_names): continue
dir2 = '%s/%s' % (dir1, cftype)
if not os.path.isdir(dir2): continue
logger.debug(' %s' % cftype)
cfdir = '%s/%s/%s/%s' % (dircalib, calibvers, detname, cftype)
listdicts = history_list_of_dicts('%s/HISTORY' % cfdir,
verbose)
#logger.debug('XXX listdicts %s' % listdicts)
count = 0
for fname in gu.get_list_of_files_in_dir(dir2):
logger.debug(' %s' % fname)
if fname == 'HISTORY': continue
if os.path.splitext(fname)[1] != '.data': continue
logger.debug(' XXX begin adding: %s %s %s %s' %
(dircalib, detname_m, cftype, fname))
add_calib_file_to_cdb(exp, dircalib, calibvers, detname_m,
cftype, fname, cfdir, listdicts,
**kwargs)
count += 1
logger.info(' converted %3d files from: %s' % (count, cfdir))
def add_data_and_two_docs(data,
exp,
det,
url=cc.URL_KRB,
krbheaders=cc.KRBHEADERS,
**kwargs):
""" Check permission and add data and document to experiment and detector data bases.
"""
logger.debug('add_data_and_two_docs kwargs: %s' % str(kwargs))
detname = pro_detector_name(det, add_shortname=True)
colname = detname
dbname_exp = mu.db_prefixed_name(exp)
dbname_det = mu.db_prefixed_name(detname)
kwargs['detector'] = detname # ex: epix10ka_000001
kwargs['shortname'] = detname # ex: epix10ka_000001
kwargs['longname'] = det # ex: epix10ka_<_uniqueid>
#kwargs['detname'] = det_name # already in kwargs ex: epixquad
resp = add_data_and_doc(data,
dbname_exp,
colname,
url=url,
krbheaders=krbheaders,
**kwargs)
if resp is None: return None
id_data_exp, id_doc_exp = resp
#logger.warning('Deployment of constants in the %s needs in expert-privilage\n' % dbname_det)
#if not kwargs.get('confirm', False):
#mu.request_confirmation()
#return id_data_exp, None, id_doc_exp, None
kwargs['id_data_exp'] = id_data_exp # override
kwargs['id_doc_exp'] = id_doc_exp # add
resp = add_data_and_doc(data,
dbname_det,
colname,
url=url,
krbheaders=krbheaders,
**kwargs)
id_data_det, id_doc_det = resp if resp is not None else (None, None)
return id_data_exp, id_data_det, id_doc_exp, id_doc_det
def test_get_random_doc_and_data_ids(det='cspad_0001'):
dbname = mu.db_prefixed_name(det)
colname = det
doc = find_doc(dbname, colname, query={'ctype':'pedestals'})
print('Pick up any doc for dbname:%s colname:%s pedestals: ' % (dbname,colname))
print('Document: %s' % str(doc))
id_doc = doc.get('_id', None)
id_data = doc.get('id_data', None)
print('_id : %s id_data : %s' % (id_doc, id_data))
return id_doc, id_data, dbname, colname
def detector_dbname(detname_short, **kwargs):
"""makes detector db name depending on suffix,
e.g. for detname_short='epixhr2x2_000001' and suffix='mytestdb'
returns 'cdb_epixhr2x2_000001_mytestdb'
"""
dbsuffix = kwargs.get('dbsuffix','')
#logger.debug('detector_dbname detname: %s dbsuffix: %s' % (detname_short, dbsuffix))
assert isinstance(dbsuffix, str)
dbname_det = mu.db_prefixed_name(detname_short)
if dbsuffix: dbname_det += '_%s'% dbsuffix
assert len(dbname_det) < 50
logger.debug('detector_dbname detname: %s dbsuffix: %s returns: %s' % (detname_short, dbsuffix, dbname_det))
return dbname_det
def add_data_and_two_docs(data, exp, det, url=cc.URL_KRB, krbheaders=cc.KRBHEADERS, **kwargs):
"""Add data and document to experiment and detector data bases.
"""
logger.debug('add_data_and_two_docs kwargs: %s' % str(kwargs))
detname = pro_detector_name(det, add_shortname=True)
colname = detname
dbname_exp = mu.db_prefixed_name(exp)
dbname_det = mu.db_prefixed_name(detname)
kwargs['detector'] = detname # ex: epix10ka_000001
kwargs['shortname'] = detname # ex: epix10ka_000001
kwargs['longname'] = det # ex: epix10ka_<_uniqueid>
#kwargs['detname'] = det_name # already in kwargs ex: epixquad
resp = add_data_and_doc(data, dbname_exp, colname, url=url, krbheaders=krbheaders, **kwargs)
if resp is None: return None
id_data_exp, id_doc_exp = resp
kwargs['id_data_exp'] = id_data_exp # override
kwargs['id_doc_exp'] = id_doc_exp # add
resp = add_data_and_doc(data, dbname_det, colname, url=url, krbheaders=krbheaders, **kwargs)
id_data_det, id_doc_det = resp if resp is not None else (None, None)
return id_data_exp, id_data_det, id_doc_exp, id_doc_det
def delall(self):
"""USED FOR DEVELOPMENT: Deletes all databases with prefix in the name.
"""
mode, kwargs = self.mode, self.kwargs
client = self.client()
prefix = dbu.db_prefixed_name('')
dbnames = dbu.database_names(client)
logger.info('Databases before "%s":\n%s' % (mode, str(dbnames)))
#confirm = kwargs.get('confirm', False)
confirm = True
for dbname in dbu.database_names(client):
if prefix in dbname:
logger.info('delete %s' % dbname)
if confirm:
dbu.delete_database(client, dbname)
if confirm:
logger.info('Databases after "%s" %s:\n%s' %
(mode, dbname, str(dbu.database_names(client))))
else:
dbu.request_confirmation()
def test_xtcav_calib_constants(fname=
'/reg/d/psdm/XPP/xpptut15/calib/Xtcav::CalibV1/XrayTransportDiagnostic.0:Opal1000.0/pedestals/101-102.data',\
add_constants_to_db=False) :
_, exp, _, cvers, detname, ctype, cfname = fname.rsplit('/',6)
resp = parse_calib_file_name(cfname)
begin, end, ext = resp if resp is not None else (None, None, None)
det = detname_conversion(detname)
run = begin
dbname_exp = dbu.db_prefixed_name(exp)
dbname_det = dbu.db_prefixed_name(det)
print('LCLS1 Xtcav calibration file: %s' % fname)
print('Parameters form path: exp:%s det:%s ctype:%s run:%s dbname_exp:%s dbname_det:%s'%\
(exp, det, ctype, run, dbname_exp, dbname_det))
#Save(ct,fname)
o1 = Load(fname)
d1 = dict_from_xtcav_calib_object(o1)
print('Xtcav calibration constants as dict:\n', d1)
if not add_constants_to_db : return
#==================================
#---- Delete databases for experiment and detector
client = dbu.connect_to_server(HOST, PORT)
print('Open client on host:%s port:%s' % (HOST, PORT))
print('Delete database %s'% dbname_exp)
dbu.delete_database(client, dbname_exp)
print('Delete database %s'% dbname_det)
dbu.delete_database(client, dbname_det)
#---- Add data to experiment and detector dbs
print('Add Xtcav constants')
kwargs = {'host' : HOST,\
'port' : PORT,\
'version' : 'V01',\
'comment' : 'test of add-retrieve xtcav constants'
}
#insert_calib_data(data, *kwargs)
dbu.insert_constants(o1, exp, det, ctype, run, time_sec='1000000000', **kwargs)
#msg = dbu.database_info(client, dbname_exp, level=10)
#print(msg)
print('Xtcav constants inserted, now retrieve them from db:%s collection:%s' % (dbname_exp, det))
db, fs = dbu.db_and_fs(client, dbname_exp)
col = dbu.collection(db, det)
#for doc in col.find() :
# print(doc)
doc = dbu.find_doc(col, query={'ctype':ctype, 'run':run})
print('Found doc:\n', doc)
o2 = dbu.get_data_for_doc(fs, doc)
d2 = dict_from_xtcav_calib_object(o2)
print('Xtcav calib object converted to dict:\n', d2)
#print('cmp(d1,d2) :', str(d1==d2))
print('\nCompare dictionaries for Xtcav calib objects loaded directly from calib file and passed through the CDB')
compare_dicts(d1,d2)
client.close()
return