def save(self, group):
grp = get_subgroup(group, self.range())
ds1 = save_object_as_dset(grp, 'begin',
data=self.begin()) # dtype='double'
ds2 = save_object_as_dset(grp, 'end',
data=self.end()) # dtype='double'
ds3 = save_object_as_dset(grp, 'range',
data=self.range()) # dtype='str'
ds4 = save_object_as_dset(grp, 'versdef',
data=self._vnum_def) # dtype='int'
msg = '=== save(), group %s object for %s' % (grp.name, self.range())
log.debug(msg, self._name)
#print 'ZZZ: self.versions()', self.versions()
# save/delete objects in/from hdf5 file
for k, v in self._dicvers.iteritems():
if k in self._lst_del_keys:
delete_object(grp, version_int_to_str(k))
else:
v.save(grp)
# deletes items from dictionary
for k in self._lst_del_keys:
del self._dicvers[k]
self._lst_del_keys = []
self.save_base(grp)
def load(self, grp):
msg = '==== load data from group %s and fill object %s' % (grp.name,
self._name)
log.debug(msg, self._name)
for k, v in dict(grp).iteritems():
#subgrp = v
#print ' ', k , v
if isinstance(v, sp.dataset_t):
log.debug('load dataset "%s"' % k, self._name)
#t0_sec = time()
if k == 'version': self.set_vnum(v[0])
elif k == 'tsprod': self.set_tsprod(v[0])
elif k == 'data':
d = v.value
if str(d.dtype)[:2] == '|S':
d = d.tostring() # .split('\n')
#print 'XXX: d, type(d): %s'%d, type(d)
self.add_data(d)
else:
log.warning(
'group "%s" has unrecognized dataset "%s"' %
(grp.name, k), self._name)
#print 'TTT %s dataset "%s" time (sec) = %.6f' % (sys._getframe().f_code.co_name, k, time()-t0_sec)
elif isinstance(v, sp.group_t):
if self.is_base_group(k, v): continue
def load(self, path=None):
with sp.File(self._fpath, 'r') as grp:
#msg = 'Load data from file %s and fill %s object for group "%s"' % (self._fpath, self._name, grp.name)
#log.info(msg, self._name)
log.info('Load data from file %s' % self._fpath, self._name)
for k, v in dict(grp).iteritems():
#subgrp = v
#print ' ', k # , " ", subg.name #, val, subg.len(), type(subg),
if isinstance(v, sp.dataset_t):
log.debug('load dataset "%s"' % k, self._name)
if k == 'dettype': self.set_dettype(v[0])
elif k == 'detid': self.set_detid(v[0])
elif k == 'detname': self.set_detname(v[0])
elif k == 'tscfile': self.set_tscfile(v[0])
elif k == 'predecessor': self.set_predecessor(v[0])
elif k == 'successor': self.set_successor(v[0])
else:
log.warning(
'hdf file has unrecognized dataset "%s"' % k,
self._name)
elif isinstance(v, sp.group_t):
if self.is_base_group(k, v): continue
log.debug('load group "%s"' % k, self._name)
o = self.add_ctype(k, cmt=False)
o.load(v)
def save_history_file(self, path='history.txt', verb=False):
"""Save history in the text file"""
f = open(path, 'w')
f.write(self.history_text())
f.close()
if verb:
#print 'History records are saved in the file: %s' % path
log.debug('History records are saved in the file: %s' % path,
self._name)
def __init__(self, path, cmt=None):
DCStoreI.__init__(self, path, cmt)
self._name = self.__class__.__name__
self._set_file_name(path)
self._tscfile = None
self._predecessor = None
self._successor = None
self._dicctypes = {}
log.debug('In c-tor for path: %s' % path, self._name)
def __init__(self, begin, end=None, cmt=None): # double, double/None
DCRangeI.__init__(self, begin, end, cmt)
self._name = self.__class__.__name__
self.set_begin(begin)
self.set_end(end)
self._dicvers = {}
self._vnum_def = 0 # 0 = use last
self._str_range = key(begin, end)
log.debug('In c-tor for range: %s' % self._str_range, self._name)
def create_directory(dir, mode=0775):
#print 'create_directory: %s' % dir
if os.path.exists(dir):
log.debug('Directory exists: %s' % dir, __name__)
else:
os.makedirs(dir, mode)
#os.chmod(dir, mode)
#os.system(cmd)
log.info('Directory created: %s' % dir, __name__)
def _load_hystory_dict(self, grp):
log.debug('_load_hystory_dict for group %s' % grp.name, self._name)
self.clear_history()
for k, v in zip(grp.keys(), grp.values()):
#print ' YYY:k,v:', k,v
tsec = float(k)
rec = 'None' if v is None else v[0]
log.debug('t: %.6f rec: %s' % (tsec, rec), self._name)
self.add_history_record(rec, tsec) # tsec=self.tstr_to_tsec(k)
def __init__(self, cmt=None):
self._name = 'DCBase'
self._dicpars = {}
self._dichist = {}
msg = 'In c-tor %s' % self._name
log.debug(msg, self._name)
self._grp_pars_name = '_parameters'
self._grp_history_name = '_history'
self._tsec_old = None
self._lst_del_keys = []
if cmt is not None: self.add_history_record(cmt)
def __init__(self,
vnum,
tsprod=None,
arr=None,
cmt=None): # int, double, np.array
DCVersionI.__init__(self, vnum, tsprod, arr, cmt)
self._name = self.__class__.__name__
self.set_vnum(vnum)
self.set_tsprod(tsprod)
self.add_data(arr)
log.debug('In c-tor for version: %s' % vnum, self._name)
def create_path(path, depth=2, mode=0775):
# Creates missing path for /reg/d/psdm/<INS>/<EXP>/calib/<dtype> beginning from calib
subdirs = path.rstrip('/').rsplit('/', depth)
log.debug('subdirs: %s' % str(subdirs), __name__)
cpath = subdirs[0]
length = len(subdirs)
for i, sd in enumerate(subdirs[1:]):
cpath += '/%s' % sd
#if i<length-depth : continue
create_directory(cpath, mode)
#print 'create_path: %s' % cpath
return os.path.exists(cpath)
def load_history_file(self, path='history.txt', verb=False):
"""Load history from the text file"""
f = open(path, 'r')
lines = f.readlines()
f.close()
for line in lines:
tsstr, rec = line.rstrip('\n').split(' ', 1)
#print 'XXX:', tsstr, rec
self._dichist[self.tstr_to_tsec(tsstr)] = rec
if verb:
#print 'Read history records from the file: %s' % path
log.debug('Read history records from the file: %s' % path,
self._name)
def save(self, group):
grp = get_subgroup(group, self.str_vnum()) # (str)
ds1 = save_object_as_dset(grp, 'version',
data=self.vnum()) # dtype='int'
ds2 = save_object_as_dset(grp, 'tsprod',
data=self.tsprod()) # dtype='double'
ds3 = save_object_as_dset(
grp, 'data', data=self.data()) # dtype='str' or 'np.array'
msg = '==== save(), group %s object for version %d' % (grp.name,
self.vnum())
log.debug(msg, self._name)
self.save_base(grp)
def save(self, path=None, mode='r+'):
if path is not None: self._fpath = path
if not isinstance(self._fpath, str):
msg = 'Invalid file name: %s' % str(self._fpath)
log.error(msg, self.__class__.__name__)
raise ValueError(msg)
mode_rw = mode if os.path.exists(self._fpath) else 'w'
with sp.File(self._fpath, mode_rw) as grp:
msg = '= save(), group %s object for %s' % (grp.name,
self.detname())
log.debug(msg, self._name)
ds1 = save_object_as_dset(grp, 'dettype',
data=self.dettype()) # 'str'
ds2 = save_object_as_dset(grp, 'detname',
data=self.detname()) # 'str'
ds3 = save_object_as_dset(grp, 'detid', data=self.detid()) # 'str'
ds4 = save_object_as_dset(grp, 'tscfile',
data=self.tscfile()) # 'double'
ds5 = save_object_as_dset(grp,
'predecessor',
data=self.predecessor()) # 'str'
ds6 = save_object_as_dset(grp, 'successor',
data=self.successor()) # 'str'
# save/delete objects in/from hdf5 file
for k, v in self._dicctypes.iteritems():
if k in self._lst_del_keys: delete_object(grp, k)
else: v.save(grp)
#self._dicctypes[k].save(grp)
# deletes items from dictionary
for k in self._lst_del_keys:
del self._dicctypes[k]
self._lst_del_keys = []
self.save_base(grp)
grp.close()
log.info('File %s is updated/saved' % self._fpath, self._name)
def save(self, group):
grp = get_subgroup(group, self.ctype())
ds1 = save_object_as_dset(grp, 'ctype',
data=self.ctype()) # dtype='str'
msg = '== save(), group %s object for %s' % (grp.name, self.ctype())
log.debug(msg, self._name)
# save/delete objects in/from hdf5 file
for k, v in self._dicranges.iteritems():
if k in self._lst_del_keys: delete_object(grp, k)
else: v.save(grp)
# deletes items from dictionary
for k in self._lst_del_keys:
del self._dicranges[k]
self._lst_del_keys = []
self.save_base(grp)
def load(self, grp):
msg = '=== load data from group %s and fill object %s' % (grp.name,
self._name)
log.debug(msg, self._name)
#print 'XXX load grp, keys:', grp, grp.keys()
for k, v in dict(grp).iteritems():
#subgrp = v
if isinstance(v, sp.dataset_t):
log.debug('load dataset "%s"' % k, self._name)
if k == 'begin': self.set_begin(v[0])
elif k == 'end': self.set_end(v[0])
elif k == 'range': self.set_str_range(v[0])
elif k == 'versdef':
self.set_vnum_def(v[0]) # self._vnum_def = v[0]
else:
log.warning(
'group "%s" has unrecognized dataset "%s"' %
(grp.name, k), self._name)
elif isinstance(v, sp.group_t):
#print ' YYY:group v.name, v.keys():', v.name, v.keys()
if self.is_base_group(k, v): continue
log.debug('load group "%s"' % k, self._name)
version = v.get('version')
if version is None:
msg = 'corrupted file structure - group "%s" does not contain key "version", keys: "%s"' % (
v.name, v.keys())
log.error(msg, self._name)
print 'ERROR:', self._name, msg
continue
o = self.add_version(version[0], cmt=False)
o.load(v)
def load(self, grp):
msg = '== load data from group %s and fill object %s' % (grp.name,
self._name)
log.debug(msg, self._name)
for k, v in dict(grp).iteritems():
#subgrp = v
#print 'XXX ', k , v# , " ", subg.name #, val, subg.len(), type(subg),
if isinstance(v, sp.dataset_t):
log.debug('load dataset "%s"' % k, self._name)
if k == 'ctype': self.set_ctype(v[0])
else:
log.warning(
'group "%s" has unrecognized dataset "%s"' %
(grp.name, k), self._name)
elif isinstance(v, sp.group_t):
if self.is_base_group(k, v): continue
log.debug('load group "%s"' % k, self._name)
begin = v.get('begin')
if begin is None:
msg = 'corrupted file structure - group "%s" does not contain key "begin", keys: "%s"' % (
v.name, v.keys())
log.error(msg, self._name)
print 'ERROR:', self._name, msg
continue
end = v.get('end')
if end is None:
msg = 'corrupted file structure - group "%s" does not contain key "end", keys: "%s"' % (
v.name, v.keys())
log.error(msg, self._name)
print 'ERROR:', self._name, msg
continue
#print 'ZZZ: name, k, v', v.name, v.keys(), v.values()
#print "XXX:v['begin'][0], v['end'][0]", v['begin'][0], v['end'][0]
o = self.add_range(begin[0], end[0], cmt=False)
o.load(v)
def __init__(self, ctype, cmt=None):
DCTypeI.__init__(self, ctype, cmt)
self._name = self.__class__.__name__
self._dicranges = {}
self._ctype = ctype
log.debug('In c-tor for ctype: %s' % ctype, self._name)
def _load_pars_dict(self, grp):
log.debug('_load_pars_dict for group %s' % grp.name, self._name)
self.clear_pars()
for k, v in dict(grp).iteritems():
log.debug('par: %s = %s' % (k, str(v[0])), self._name)
self.add_par(k, v[0])
