def move_recs_to_archive(procname, exp, runs):
"""Move expired run records from log file to archive file.
"""
fname_log = log_file(exp, procname)
fname_arc = arc_file(exp, procname)
print('Move records for old runs to archive file: %s\n' % fname_arc)
recs = gu.load_textfile(fname_log).split('\n')
recs_log = [rec for rec in recs if not (rec[:4] in runs)]
recs_arc = [rec for rec in recs if rec[:4] in runs]
text_log = '\n'.join(recs_log)
text_log += '\n'
#if len(runs)==1 : text_log+='\n'
text_arc = '\n'.join(recs_arc)
text_arc += '\n'
#if len(runs)==1 : text_arc+='\n'
#print(' ==> log\n%s' % text_log)
#print(' ==> arc\n%s' % text_arc)
gu.save_textfile(text_log, fname_log, mode='w')
os.chmod(fname_log, 0o664)
gu.save_textfile(text_arc, fname_arc, mode='a')
os.chmod(fname_log, 0o664)
def add_calib_file_to_cdb(exp, dircalib, calibvers, detname, cftype, fname,
cfdir, listdicts, **kwargs):
"""
"""
d = history_dict_for_file(listdicts, fname)
resp = parse_calib_file_name(fname)
begin, end, ext = resp if resp is not None else (None, None, None)
if begin is not None: begin = int(begin)
if None in (begin, end, ext): return
fpath = '%s/%s' % (cfdir, fname)
verbose = kwargs.get('verbose', False)
data = gu.load_textfile(fpath, verbose) if cftype in ('geometry','code_geometry') else\
load_xtcav_calib_file(fpath) if is_xtcav(calibvers, cftype) else\
load_txt(fpath) # using NDArrIO
if isinstance(data, dict):
serialize_dict(data)
logger.debug(info_dict(data))
#print_dict(data)
#data = json.dumps(data) # (data,ensure_ascii=True) json.dumps converts dict -> str # .replace("'", '"') for json
data = str(data)
if isinstance(data, np.ndarray):
check_data_shape(data, detname, cftype)
begin_time, end_time = run_begin_end_time(exp, int(begin))
if verbose:
ndu.print_ndarr(data, 'scan calib: data')
msg = 'scan calib: %s %s %s %s %s %s %s %s %s' % (
exp, cfdir, fname, begin, end, ext, calibvers, detname, cftype)
logger.info(msg)
logger.info('begin_time: %s end_time: %s' % (begin_time, end_time))
if data is None:
msg = 'data is None, conversion is dropped for for file: %s' % fpath
logger.warning(msg)
return
kwargs['run'] = begin
kwargs['run_end'] = end
kwargs['detector'] = detname
kwargs['ctype'] = cftype
kwargs['time_sec'] = begin_time
kwargs['end_time'] = end_time
kwargs['time_stamp'] = dbu._timestamp(begin_time)
kwargs['extpars'] = d if d is not None else {
} # just in case save entire history dict
#kwargs['comment'] = 'HISTORY: %s' % d.get('comment', '')
dbu.insert_calib_data(data, **kwargs)
def experiments_under_control(procname='pixel_status'):
"""Returns list of (str) experiment names from control file.
"""
fname = control_file(procname)
if not os.path.lexists(fname):
#raise IOError('Control file "%s" does not exist' % fname)
print('WARNING: control file "%s" does not exist' % fname)
return []
recs = gu.load_textfile(fname).split('\n')
return [rec for rec in recs
if (rec and (rec[0] != '#'))] # skip empty and commented records
def recs_in_log_file(exp='xpptut15', procname='pixel_status', verb=0):
"""Returns list of (str) records in the log file for specified experiment and process name.
E.g. of one record: '0151 2017-10-05T15:19:21'
"""
fname_log = log_file(exp, procname)
if verb: print('Log file: %s' % fname_log)
if not os.path.lexists(fname_log):
if verb: print('Log file "%s" does not exist' % fname_log)
return []
recs = gu.load_textfile(fname_log).split('\n')
return recs # list of records, each record is '0059 <time-stamp>'
def change_value(self, item, key, path):
logger.debug('change_value for key: %s' % (key))
if key == self.data_fname:
item.setText(str(path))
self.data_nda = self.load_nda_from_file(path)
logger.info(info_ndarr(self.data_nda, 'From file %s loaded array' % path))
self.set_metadata_values()
item.setBackground(QBrush(Qt.cyan))
else:
txt = gu.load_textfile(path)
logger.info('From file %s fill field: %s' % (path,txt))
item.setText(txt)
def add(self):
"""Adds calibration constants to database from file.
"""
kwargs = self.kwargs
fname = kwargs.get('iofname', 'None')
ctype = kwargs.get('ctype', 'None')
assert os.path.exists(fname), 'File "%s" DOES NOT EXIST' % fname
ext = os.path.splitext(fname)[-1]
data = gu.load_textfile(fname, verb=False) if ctype == 'geometry' else\
np.load(fname) if ext == '.npy' else\
load_txt(fname)
dbu.insert_calib_data(data, **kwargs)
def add(self):
"""Adds calibration constants to database from file.
"""
kwargs = self.kwargs
fname = kwargs.get('iofname', 'None')
ctype = kwargs.get('ctype', 'None')
dtype = kwargs.get('dtype', 'None')
verb = self.loglevel == 'DEBUG'
assert os.path.exists(fname), 'File "%s" DOES NOT EXIST' % fname
ext = os.path.splitext(fname)[-1]
data = gu.load_textfile(fname, verb=verb) if ctype == 'geometry' or dtype in ('str', 'txt', 'text') else\
load_xtcav_calib_file(fname) if dtype == 'xtcav' else\
np.load(fname) if ext == '.npy' else\
gu.load_json(fname) if ext == '.json' or dtype == 'json' else\
gu.load_pickle(fname) if ext == '.pkl' or dtype in ('pkl', 'pickle') else\
load_txt(fname) # input NDArrIO
dbu.insert_calib_data(data, **kwargs)
def add_calib_file_to_cdb(exp, dircalib, calibvers, detname, cftype, fname, cfdir, listdicts, **kwargs) :
"""
"""
d = history_dict_for_file(listdicts, fname)
resp = parse_calib_file_name(fname)
begin, end, ext = resp if resp is not None else (None, None, None)
if None in (begin, end, ext) : return
fpath = '%s/%s' % (cfdir, fname)
verbose = kwargs.get('verbose', False)
data = gu.load_textfile(fpath, verbose) if cftype in ('geometry','code_geometry') else\
load_xtcav_calib_file(fpath) if is_xtcav(calibvers, cftype) else\
load_txt(fpath) # using NDArrIO
begin_time, end_time = run_begin_end_time(exp, int(begin))
if verbose :
ndu.print_ndarr(data, 'scan calib: data')
msg = 'scan calib: %s %s %s %s %s %s %s %s %s' % (exp, cfdir, fname, begin, end, ext, calibvers, detname, cftype)
logger.info(msg)
logger.info('begin_time: %s end_time: %s' % (begin_time, end_time))
if data is None :
msg = 'data is None, conversion is dropped for for file: %s' % fpath
logger.warning(msg)
return
kwargs['run'] = begin
kwargs['run_end'] = end
kwargs['detector'] = detname
kwargs['ctype'] = cftype
kwargs['time_sec'] = begin_time
kwargs['end_time'] = end_time
kwargs['time_stamp'] = dbu._timestamp(begin_time)
kwargs['extpars'] = d # just in case save entire history dict
#kwargs['comment'] = 'HISTORY: %s' % d.get('comment', '')
dbu.insert_calib_data(data, **kwargs)
def text_data(fname):
logger.info('DLDUtils.text_data from file') #: %s' % fname)
data = gu.load_textfile(fname, verb=True)
logger.debug(data)
return data
def history_records(history, verb=True):
"""Returns list of (str) records from history file.
"""
if not os.path.exists(history): return None
return gu.load_textfile(history, verb).split('\n')