def deldoc(self):
"""Deletes specified document in the database.
"""
mode, kwargs = self.mode, self.kwargs
dbname = dbu.get_dbname(**kwargs)
client = self.client()
if not self.check_database(client, dbname): return
detname = kwargs.get('detector', None)
if detname is None:
logger.warning(
'%s needs in the collection name. Please specify the detector name.'
% (mode))
colname = detname
db, fs = dbu.db_and_fs(client, dbname)
colnames = dbu.collection_names(db)
if not (colname in colnames): # dbu.collection_exists(db, colname)
logger.warning('db "%s" does not have collection "%s"' %
(db.name, str(colname)))
return
col = dbu.collection(db, colname)
logger.info('command mode: "%s" db: "%s" collection: "%s"' %
(mode, db.name, str(colname)))
defs = self.defs
ctype = kwargs.get('ctype', None)
run = kwargs.get('run', None)
tsec = kwargs.get('time_sec', None)
tstamp = kwargs.get('time_stamp', None)
vers = kwargs.get('version', None)
confirm = kwargs.get('confirm', False)
query = {'detector': detname}
if ctype != defs['ctype']: query['ctype'] = ctype
if run != defs['run']: query['run'] = run
if vers != defs['version']: query['version'] = vers
#if tsec != defs['time_sec'] : query['time_sec'] = tsec
if is_in_command_line('-s', '--time_sec'): query['time_sec'] = tsec
if is_in_command_line('-t', '--time_stamp'):
query['time_stamp'] = tstamp
logger.info('query: %s' % str(query))
docs = dbu.find_docs(col, query)
if docs is None or docs.count() == 0:
logger.warning('Can not find document for query: %s' % str(query))
return
for i, doc in enumerate(docs):
msg = ' deldoc %2d:'%i + doc['time_stamp'] + ' ' + str(doc['time_sec'])\
+ ' %s'%doc['ctype'].ljust(16) + ' %4d'%doc['run'] + ' ' + str(doc['id_data'])
logger.info(msg)
if confirm:
dbu.delete_document_from_collection(col, doc['_id'])
dbu.del_document_data(doc, fs)
if not confirm: dbu.request_confirmation()
def delete_documents(dbname, colname, doc_ids):
"""Delete documents with _id-s in doc_ids from dbname, colname
"""
#logger.debug('Deleting documents:\n %s' % ('\n '.join(doc_ids)))
client = connect_client()
db, fs = dbu.db_and_fs(client, dbname)
col = collection(db, colname)
#msg = 'Deleted documents from db: %s col: %s' % (dbname, colname)
for s in doc_ids:
oid = ObjectId(s)
doc = dbu.find_doc(col, query={'_id': oid})
if doc is None: continue
#msg += '\n %s and its data' % doc.get('_id', 'N/A')
dbu.del_document_data(doc, fs)
dbu.delete_document_from_collection(col, oid)
def delcol(self):
"""Deletes specified collection in the database.
"""
mode, kwargs = self.mode, self.kwargs
dbname = dbu.get_dbname(**self.kwargs)
client = self.client()
if not self.check_database(client, dbname): return
detname = kwargs.get('detector', None)
if detname is None:
logger.warning(
'%s needs in the collection name. Please specify the detector name.'
% (mode))
colname = detname
db, fs = dbu.db_and_fs(client, dbname)
colnames = dbu.collection_names(db)
logger.info('"%s" deletes collection "%s" from database "%s"' %
(mode, colname, db.name))
logger.info('Collections before "%s"' % str(colnames))
logger.info(dbu.database_fs_info(db, gap=''))
if not (colname in colnames):
logger.warning('db "%s" does not have collection "%s"' %
(db.name, str(colname)))
return
if kwargs.get('confirm', False):
col = dbu.collection(db, colname)
dbu.del_collection_data(
col, fs) # delete data in fs associated with collection col
dbu.delete_collection_obj(col)
logger.info('Collections after "%s"' %
str(dbu.collection_names(db)))
logger.info(dbu.database_fs_info(db, gap=''))
else:
dbu.request_confirmation()
def get_data_for_doc(dbname, doc):
client = connect_client()
db, fs = dbu.db_and_fs(client, dbname)
return dbu.get_data_for_doc(fs, doc)
def insert_document_and_data(dbname, colname, doc, data):
client = connect_client()
db, fs = dbu.db_and_fs(client, dbname)
col = collection(db, colname)
id_data, id_doc = dbu.insert_data_and_doc(data, fs, col, **doc)
return id_data, id_doc
def test_xtcav_calib_constants(fname=
'/reg/d/psdm/XPP/xpptut15/calib/Xtcav::CalibV1/XrayTransportDiagnostic.0:Opal1000.0/pedestals/101-102.data',\
add_constants_to_db=False) :
_, exp, _, cvers, detname, ctype, cfname = fname.rsplit('/',6)
resp = parse_calib_file_name(cfname)
begin, end, ext = resp if resp is not None else (None, None, None)
det = detname_conversion(detname)
run = begin
dbname_exp = dbu.db_prefixed_name(exp)
dbname_det = dbu.db_prefixed_name(det)
print('LCLS1 Xtcav calibration file: %s' % fname)
print('Parameters form path: exp:%s det:%s ctype:%s run:%s dbname_exp:%s dbname_det:%s'%\
(exp, det, ctype, run, dbname_exp, dbname_det))
#Save(ct,fname)
o1 = Load(fname)
d1 = dict_from_xtcav_calib_object(o1)
print('Xtcav calibration constants as dict:\n', d1)
if not add_constants_to_db : return
#==================================
#---- Delete databases for experiment and detector
client = dbu.connect_to_server(HOST, PORT)
print('Open client on host:%s port:%s' % (HOST, PORT))
print('Delete database %s'% dbname_exp)
dbu.delete_database(client, dbname_exp)
print('Delete database %s'% dbname_det)
dbu.delete_database(client, dbname_det)
#---- Add data to experiment and detector dbs
print('Add Xtcav constants')
kwargs = {'host' : HOST,\
'port' : PORT,\
'version' : 'V01',\
'comment' : 'test of add-retrieve xtcav constants'
}
#insert_calib_data(data, *kwargs)
dbu.insert_constants(o1, exp, det, ctype, run, time_sec='1000000000', **kwargs)
#msg = dbu.database_info(client, dbname_exp, level=10)
#print(msg)
print('Xtcav constants inserted, now retrieve them from db:%s collection:%s' % (dbname_exp, det))
db, fs = dbu.db_and_fs(client, dbname_exp)
col = dbu.collection(db, det)
#for doc in col.find() :
# print(doc)
doc = dbu.find_doc(col, query={'ctype':ctype, 'run':run})
print('Found doc:\n', doc)
o2 = dbu.get_data_for_doc(fs, doc)
d2 = dict_from_xtcav_calib_object(o2)
print('Xtcav calib object converted to dict:\n', d2)
#print('cmp(d1,d2) :', str(d1==d2))
print('\nCompare dictionaries for Xtcav calib objects loaded directly from calib file and passed through the CDB')
compare_dicts(d1,d2)
client.close()
return
def get(self):
"""Saves in file calibration constants from database.
"""
mode, kwargs = self.mode, self.kwargs
defs = self.defs
host = kwargs.get('host', None)
port = kwargs.get('port', None)
exp = kwargs.get('experiment', None)
det = kwargs.get('detector', None)
ctype = kwargs.get('ctype', None)
run = kwargs.get('run', None)
run_end = kwargs.get('run_end', None)
tsec = kwargs.get('time_sec', None)
tstamp = kwargs.get('time_stamp', None)
vers = kwargs.get('version', None)
fname = kwargs.get('iofname', None)
query = {'detector': det, 'ctype': ctype}
if run != defs['run']:
query['run'] = {'$lte': run}
query['run_end'] = {'$gte': run}
if tsec != defs['time_sec']: query['time_sec'] = {'$lte': tsec}
if vers != defs['version']: query['version'] = vers
#logger.debug('query: %s' % str(query))
dbname = dbu.get_dbname(**kwargs)
client = self.client()
if not self.check_database(client, dbname): return
detname = kwargs.get('detector', None)
if detname is None:
logger.warning(
'%s needs in the collection name. Please specify the detector name.'
% (mode))
colname = detname
db, fs = dbu.db_and_fs(client, dbname)
colnames = dbu.collection_names(db)
if not (colname in colnames): # dbu.collection_exists(db, colname)
logger.warning('db "%s" does not have collection "%s"' %
(db.name, str(colname)))
return
col = dbu.collection(db, colname)
logger.info('Search document in db "%s" collection "%s"' %
(dbname, colname))
#client, expname, detname, db_exp, db_det, fs_exp, fs_det, col_exp, col_det =\
# dbu.connect(host=host, port=port, experiment=exp, detector=det, verbose=verb)
#fs, doc = fs_exp, dbu.find_doc(col, query)
#if doc is None :
# fs, doc = fs_det, dbu.find_doc(col_det, query)
# if doc is None :
# logger.warning('Can not find document for query: %s' % str(query))
# return
doc = dbu.find_doc(col, query)
if doc is None:
logger.warning('Can not find document for query: %s' % str(query))
return
data = dbu.get_data_for_doc(fs, doc)
if data is None:
logger.warning('Can not load data for doc: %s' % str(doc))
return
if fname is None: fname = 'clb-%s-%s-%s.npy' % (expname, det, ctype)
if ctype == 'geometry':
gu.save_textfile(data, fname, mode='w', verb=verb)
elif os.path.splitext(fname)[1] == '.npy':
np.save(fname, data, allow_pickle=False)
else:
save_txt(fname, data, fmt='%.2f')
if verb: logger.info(info_ndarr(data, 'nda', first=0, last=3))
logger.info('Save constants in file: %s' % fname)
def get(self):
"""Gets constans from DB and saves them in file.
"""
mode, kwargs = self.mode, self.kwargs
defs = self.defs
host = kwargs.get('host', None)
port = kwargs.get('port', None)
exp = kwargs.get('experiment', None)
det = kwargs.get('detector', None)
ctype = kwargs.get('ctype', None)
run = kwargs.get('run', None)
run_end = kwargs.get('run_end', None)
tsec = kwargs.get('time_sec', None) if is_in_command_line(
'-s', '--time_sec') else None
tstamp = kwargs.get('time_stamp', None) if is_in_command_line(
'-t', '--time_stamp') else None
vers = kwargs.get('version', None)
prefix = kwargs.get('iofname', None)
verb = self.loglevel == 'DEBUG'
db_det, db_exp, colname, query = dbu.dbnames_collection_query(
det, exp, ctype, run, tsec, vers)
logger.debug('get: %s %s %s %s' %
(db_det, db_exp, colname, str(query)))
dbname = db_det if exp is None else db_exp
client = self.client()
if not self.check_database(client, dbname): return
db, fs = dbu.db_and_fs(client, dbname)
colnames = dbu.collection_names(db)
if not (colname in colnames): # dbu.collection_exists(db, colname)
logger.warning('db "%s" does not have collection "%s"' %
(db.name, str(colname)))
return
col = dbu.collection(db, colname)
logger.debug('Search document in db "%s" collection "%s"' %
(dbname, colname))
doc = dbu.find_doc(col, query)
if doc is None:
logger.warning('Can not find document for query: %s' % str(query))
return
logger.debug('get doc:', doc)
data = dbu.get_data_for_doc(fs, doc)
if data is None:
logger.warning('Can not load data for doc: %s' % str(doc))
return
if prefix is None: prefix = dbu.out_fname_prefix(**doc)
dbu.save_doc_and_data_in_file(doc,
data,
prefix,
control={
'data': True,
'meta': True
})