def load(self, grp):
msg = '==== load data from group %s and fill object %s' % (grp.name,
self._name)
log.debug(msg, self._name)
for k, v in dict(grp).iteritems():
#subgrp = v
#print ' ', k , v
if isinstance(v, sp.dataset_t):
log.debug('load dataset "%s"' % k, self._name)
#t0_sec = time()
if k == 'version': self.set_vnum(v[0])
elif k == 'tsprod': self.set_tsprod(v[0])
elif k == 'data':
d = v.value
if str(d.dtype)[:2] == '|S':
d = d.tostring() # .split('\n')
#print 'XXX: d, type(d): %s'%d, type(d)
self.add_data(d)
else:
log.warning(
'group "%s" has unrecognized dataset "%s"' %
(grp.name, k), self._name)
#print 'TTT %s dataset "%s" time (sec) = %.6f' % (sys._getframe().f_code.co_name, k, time()-t0_sec)
elif isinstance(v, sp.group_t):
if self.is_base_group(k, v): continue
def load(self, path=None):
with sp.File(self._fpath, 'r') as grp:
#msg = 'Load data from file %s and fill %s object for group "%s"' % (self._fpath, self._name, grp.name)
#log.info(msg, self._name)
log.info('Load data from file %s' % self._fpath, self._name)
for k, v in dict(grp).iteritems():
#subgrp = v
#print ' ', k # , " ", subg.name #, val, subg.len(), type(subg),
if isinstance(v, sp.dataset_t):
log.debug('load dataset "%s"' % k, self._name)
if k == 'dettype': self.set_dettype(v[0])
elif k == 'detid': self.set_detid(v[0])
elif k == 'detname': self.set_detname(v[0])
elif k == 'tscfile': self.set_tscfile(v[0])
elif k == 'predecessor': self.set_predecessor(v[0])
elif k == 'successor': self.set_successor(v[0])
else:
log.warning(
'hdf file has unrecognized dataset "%s"' % k,
self._name)
elif isinstance(v, sp.group_t):
if self.is_base_group(k, v): continue
log.debug('load group "%s"' % k, self._name)
o = self.add_ctype(k, cmt=False)
o.load(v)
def load(self, grp):
msg = '=== load data from group %s and fill object %s' % (grp.name,
self._name)
log.debug(msg, self._name)
#print 'XXX load grp, keys:', grp, grp.keys()
for k, v in dict(grp).iteritems():
#subgrp = v
if isinstance(v, sp.dataset_t):
log.debug('load dataset "%s"' % k, self._name)
if k == 'begin': self.set_begin(v[0])
elif k == 'end': self.set_end(v[0])
elif k == 'range': self.set_str_range(v[0])
elif k == 'versdef':
self.set_vnum_def(v[0]) # self._vnum_def = v[0]
else:
log.warning(
'group "%s" has unrecognized dataset "%s"' %
(grp.name, k), self._name)
elif isinstance(v, sp.group_t):
#print ' YYY:group v.name, v.keys():', v.name, v.keys()
if self.is_base_group(k, v): continue
log.debug('load group "%s"' % k, self._name)
version = v.get('version')
if version is None:
msg = 'corrupted file structure - group "%s" does not contain key "version", keys: "%s"' % (
v.name, v.keys())
log.error(msg, self._name)
print 'ERROR:', self._name, msg
continue
o = self.add_version(version[0], cmt=False)
o.load(v)
def save_object_as_dset(grp, name, shape=None, dtype=None, data=0):
"""Saves object as h5py dataset
Currently supports scalar int, double, string and numpy.array
"""
#print 'XXX: save_object_as_dset '
#print 'XXX grp.keys():', grp.keys()
#print 'XXX %s in grp.keys(): ' % name, name in grp.keys()
if name in grp.keys(): return
if isinstance(data, np.ndarray):
return grp.create_dataset(name, data=data)
sh = (1, ) if shape is None else shape
if dtype is not None:
return grp.create_dataset(name, shape=sh, dtype=dtype, data=data)
if isinstance(data, str):
return save_string_as_dset(grp, name, data)
if isinstance(data, int):
return grp.create_dataset(name, shape=sh, dtype='int', data=data)
if isinstance(data, float):
return grp.create_dataset(name, shape=sh, dtype='double', data=data)
log.warning(
"Can't save parameter: %s of %s in the h5py group: %s" %
(name, str(dtype), grp.name), 'DCUtils.save_object_as_dset')
def set_vnum_def(self, vnum=None):
if vnum is None or vnum == 0:
self._vnum_def = 0 # will use last
elif vnum in self._dicvers.keys():
self._vnum_def = vnum
self.add_history_record(
'WARNING: set_vnum_defdef sets default version %d' % vnum)
else:
msg = 'Attemt to set non-existent version %d as default' % vnum
log.warning(msg, self._name)
def mark_ctype(self, ctype, cmt=False):
"""Marks child object for deletion in save()"""
if ctype in self._dicctypes.keys():
self._lst_del_keys.append(ctype)
rec = self.make_record('del ctype', ctype, cmt)
if cmt is not False: self.add_history_record(rec)
log.info(rec, self.__class__.__name__)
return ctype
else:
msg = 'Marking of non-existent ctype "%s"' % str(ctype)
log.warning(msg, self._name)
return None
def mark_version(self, vnum=None, cmt=False):
"""Marks child object for deletion in save()"""
vers = self.vnum_last() if vnum is None else vnum
if vers in self._dicvers.keys():
self._lst_del_keys.append(vers)
rec = self.make_record('del version', str(vers), cmt)
if cmt is not False: self.add_history_record(rec)
log.info(rec, self.__class__.__name__)
return vers
else:
msg = 'Marking of non-existent version %s' % str(vers)
log.warning(msg, self._name)
return None
def mark_range_for_key(self, keyrng, cmt=False):
"""Marks child object for deletion in save()"""
if keyrng in self._dicranges.keys():
#o = self._dicranges[keyrng]
#o.mark_versions()
self._lst_del_keys.append(keyrng)
rec = self.make_record('del range', keyrng, cmt)
if cmt is not False: self.add_history_record(rec)
log.info(rec, self.__class__.__name__)
return keyrng
else:
msg = 'Marking of non-existent range %s' % str(keyrng)
log.warning(msg, self._name)
return None
def load(self, grp):
msg = '== load data from group %s and fill object %s' % (grp.name,
self._name)
log.debug(msg, self._name)
for k, v in dict(grp).iteritems():
#subgrp = v
#print 'XXX ', k , v# , " ", subg.name #, val, subg.len(), type(subg),
if isinstance(v, sp.dataset_t):
log.debug('load dataset "%s"' % k, self._name)
if k == 'ctype': self.set_ctype(v[0])
else:
log.warning(
'group "%s" has unrecognized dataset "%s"' %
(grp.name, k), self._name)
elif isinstance(v, sp.group_t):
if self.is_base_group(k, v): continue
log.debug('load group "%s"' % k, self._name)
begin = v.get('begin')
if begin is None:
msg = 'corrupted file structure - group "%s" does not contain key "begin", keys: "%s"' % (
v.name, v.keys())
log.error(msg, self._name)
print 'ERROR:', self._name, msg
continue
end = v.get('end')
if end is None:
msg = 'corrupted file structure - group "%s" does not contain key "end", keys: "%s"' % (
v.name, v.keys())
log.error(msg, self._name)
print 'ERROR:', self._name, msg
continue
#print 'ZZZ: name, k, v', v.name, v.keys(), v.values()
#print "XXX:v['begin'][0], v['end'][0]", v['begin'][0], v['end'][0]
o = self.add_range(begin[0], end[0], cmt=False)
o.load(v)
def print_warning(obj, metframe):
wng = 'INFO: %s.%s - abstract interface method needs to be re-implemented in derived class.' \
% (obj.__class__.__name__, metframe.f_code.co_name)
log.warning(wng, obj.__class__.__name__)