def save(self, group): grp = get_subgroup(group, self.range()) ds1 = save_object_as_dset(grp, 'begin', data=self.begin()) # dtype='double' ds2 = save_object_as_dset(grp, 'end', data=self.end()) # dtype='double' ds3 = save_object_as_dset(grp, 'range', data=self.range()) # dtype='str' ds4 = save_object_as_dset(grp, 'versdef', data=self._vnum_def) # dtype='int' msg = '=== save(), group %s object for %s' % (grp.name, self.range()) log.debug(msg, self._name) #print 'ZZZ: self.versions()', self.versions() # save/delete objects in/from hdf5 file for k, v in self._dicvers.iteritems(): if k in self._lst_del_keys: delete_object(grp, version_int_to_str(k)) else: v.save(grp) # deletes items from dictionary for k in self._lst_del_keys: del self._dicvers[k] self._lst_del_keys = [] self.save_base(grp)
def load(self, grp): msg = '==== load data from group %s and fill object %s' % (grp.name, self._name) log.debug(msg, self._name) for k, v in dict(grp).iteritems(): #subgrp = v #print ' ', k , v if isinstance(v, sp.dataset_t): log.debug('load dataset "%s"' % k, self._name) #t0_sec = time() if k == 'version': self.set_vnum(v[0]) elif k == 'tsprod': self.set_tsprod(v[0]) elif k == 'data': d = v.value if str(d.dtype)[:2] == '|S': d = d.tostring() # .split('\n') #print 'XXX: d, type(d): %s'%d, type(d) self.add_data(d) else: log.warning( 'group "%s" has unrecognized dataset "%s"' % (grp.name, k), self._name) #print 'TTT %s dataset "%s" time (sec) = %.6f' % (sys._getframe().f_code.co_name, k, time()-t0_sec) elif isinstance(v, sp.group_t): if self.is_base_group(k, v): continue
def load(self, path=None): with sp.File(self._fpath, 'r') as grp: #msg = 'Load data from file %s and fill %s object for group "%s"' % (self._fpath, self._name, grp.name) #log.info(msg, self._name) log.info('Load data from file %s' % self._fpath, self._name) for k, v in dict(grp).iteritems(): #subgrp = v #print ' ', k # , " ", subg.name #, val, subg.len(), type(subg), if isinstance(v, sp.dataset_t): log.debug('load dataset "%s"' % k, self._name) if k == 'dettype': self.set_dettype(v[0]) elif k == 'detid': self.set_detid(v[0]) elif k == 'detname': self.set_detname(v[0]) elif k == 'tscfile': self.set_tscfile(v[0]) elif k == 'predecessor': self.set_predecessor(v[0]) elif k == 'successor': self.set_successor(v[0]) else: log.warning( 'hdf file has unrecognized dataset "%s"' % k, self._name) elif isinstance(v, sp.group_t): if self.is_base_group(k, v): continue log.debug('load group "%s"' % k, self._name) o = self.add_ctype(k, cmt=False) o.load(v)
def save_history_file(self, path='history.txt', verb=False): """Save history in the text file""" f = open(path, 'w') f.write(self.history_text()) f.close() if verb: #print 'History records are saved in the file: %s' % path log.debug('History records are saved in the file: %s' % path, self._name)
def __init__(self, path, cmt=None): DCStoreI.__init__(self, path, cmt) self._name = self.__class__.__name__ self._set_file_name(path) self._tscfile = None self._predecessor = None self._successor = None self._dicctypes = {} log.debug('In c-tor for path: %s' % path, self._name)
def __init__(self, begin, end=None, cmt=None): # double, double/None DCRangeI.__init__(self, begin, end, cmt) self._name = self.__class__.__name__ self.set_begin(begin) self.set_end(end) self._dicvers = {} self._vnum_def = 0 # 0 = use last self._str_range = key(begin, end) log.debug('In c-tor for range: %s' % self._str_range, self._name)
def create_directory(dir, mode=0775): #print 'create_directory: %s' % dir if os.path.exists(dir): log.debug('Directory exists: %s' % dir, __name__) else: os.makedirs(dir, mode) #os.chmod(dir, mode) #os.system(cmd) log.info('Directory created: %s' % dir, __name__)
def _load_hystory_dict(self, grp): log.debug('_load_hystory_dict for group %s' % grp.name, self._name) self.clear_history() for k, v in zip(grp.keys(), grp.values()): #print ' YYY:k,v:', k,v tsec = float(k) rec = 'None' if v is None else v[0] log.debug('t: %.6f rec: %s' % (tsec, rec), self._name) self.add_history_record(rec, tsec) # tsec=self.tstr_to_tsec(k)
def __init__(self, cmt=None): self._name = 'DCBase' self._dicpars = {} self._dichist = {} msg = 'In c-tor %s' % self._name log.debug(msg, self._name) self._grp_pars_name = '_parameters' self._grp_history_name = '_history' self._tsec_old = None self._lst_del_keys = [] if cmt is not None: self.add_history_record(cmt)
def __init__(self, vnum, tsprod=None, arr=None, cmt=None): # int, double, np.array DCVersionI.__init__(self, vnum, tsprod, arr, cmt) self._name = self.__class__.__name__ self.set_vnum(vnum) self.set_tsprod(tsprod) self.add_data(arr) log.debug('In c-tor for version: %s' % vnum, self._name)
def create_path(path, depth=2, mode=0775): # Creates missing path for /reg/d/psdm/<INS>/<EXP>/calib/<dtype> beginning from calib subdirs = path.rstrip('/').rsplit('/', depth) log.debug('subdirs: %s' % str(subdirs), __name__) cpath = subdirs[0] length = len(subdirs) for i, sd in enumerate(subdirs[1:]): cpath += '/%s' % sd #if i<length-depth : continue create_directory(cpath, mode) #print 'create_path: %s' % cpath return os.path.exists(cpath)
def load_history_file(self, path='history.txt', verb=False): """Load history from the text file""" f = open(path, 'r') lines = f.readlines() f.close() for line in lines: tsstr, rec = line.rstrip('\n').split(' ', 1) #print 'XXX:', tsstr, rec self._dichist[self.tstr_to_tsec(tsstr)] = rec if verb: #print 'Read history records from the file: %s' % path log.debug('Read history records from the file: %s' % path, self._name)
def save(self, group): grp = get_subgroup(group, self.str_vnum()) # (str) ds1 = save_object_as_dset(grp, 'version', data=self.vnum()) # dtype='int' ds2 = save_object_as_dset(grp, 'tsprod', data=self.tsprod()) # dtype='double' ds3 = save_object_as_dset( grp, 'data', data=self.data()) # dtype='str' or 'np.array' msg = '==== save(), group %s object for version %d' % (grp.name, self.vnum()) log.debug(msg, self._name) self.save_base(grp)
def save(self, path=None, mode='r+'): if path is not None: self._fpath = path if not isinstance(self._fpath, str): msg = 'Invalid file name: %s' % str(self._fpath) log.error(msg, self.__class__.__name__) raise ValueError(msg) mode_rw = mode if os.path.exists(self._fpath) else 'w' with sp.File(self._fpath, mode_rw) as grp: msg = '= save(), group %s object for %s' % (grp.name, self.detname()) log.debug(msg, self._name) ds1 = save_object_as_dset(grp, 'dettype', data=self.dettype()) # 'str' ds2 = save_object_as_dset(grp, 'detname', data=self.detname()) # 'str' ds3 = save_object_as_dset(grp, 'detid', data=self.detid()) # 'str' ds4 = save_object_as_dset(grp, 'tscfile', data=self.tscfile()) # 'double' ds5 = save_object_as_dset(grp, 'predecessor', data=self.predecessor()) # 'str' ds6 = save_object_as_dset(grp, 'successor', data=self.successor()) # 'str' # save/delete objects in/from hdf5 file for k, v in self._dicctypes.iteritems(): if k in self._lst_del_keys: delete_object(grp, k) else: v.save(grp) #self._dicctypes[k].save(grp) # deletes items from dictionary for k in self._lst_del_keys: del self._dicctypes[k] self._lst_del_keys = [] self.save_base(grp) grp.close() log.info('File %s is updated/saved' % self._fpath, self._name)
def save(self, group): grp = get_subgroup(group, self.ctype()) ds1 = save_object_as_dset(grp, 'ctype', data=self.ctype()) # dtype='str' msg = '== save(), group %s object for %s' % (grp.name, self.ctype()) log.debug(msg, self._name) # save/delete objects in/from hdf5 file for k, v in self._dicranges.iteritems(): if k in self._lst_del_keys: delete_object(grp, k) else: v.save(grp) # deletes items from dictionary for k in self._lst_del_keys: del self._dicranges[k] self._lst_del_keys = [] self.save_base(grp)
def load(self, grp): msg = '=== load data from group %s and fill object %s' % (grp.name, self._name) log.debug(msg, self._name) #print 'XXX load grp, keys:', grp, grp.keys() for k, v in dict(grp).iteritems(): #subgrp = v if isinstance(v, sp.dataset_t): log.debug('load dataset "%s"' % k, self._name) if k == 'begin': self.set_begin(v[0]) elif k == 'end': self.set_end(v[0]) elif k == 'range': self.set_str_range(v[0]) elif k == 'versdef': self.set_vnum_def(v[0]) # self._vnum_def = v[0] else: log.warning( 'group "%s" has unrecognized dataset "%s"' % (grp.name, k), self._name) elif isinstance(v, sp.group_t): #print ' YYY:group v.name, v.keys():', v.name, v.keys() if self.is_base_group(k, v): continue log.debug('load group "%s"' % k, self._name) version = v.get('version') if version is None: msg = 'corrupted file structure - group "%s" does not contain key "version", keys: "%s"' % ( v.name, v.keys()) log.error(msg, self._name) print 'ERROR:', self._name, msg continue o = self.add_version(version[0], cmt=False) o.load(v)
def load(self, grp): msg = '== load data from group %s and fill object %s' % (grp.name, self._name) log.debug(msg, self._name) for k, v in dict(grp).iteritems(): #subgrp = v #print 'XXX ', k , v# , " ", subg.name #, val, subg.len(), type(subg), if isinstance(v, sp.dataset_t): log.debug('load dataset "%s"' % k, self._name) if k == 'ctype': self.set_ctype(v[0]) else: log.warning( 'group "%s" has unrecognized dataset "%s"' % (grp.name, k), self._name) elif isinstance(v, sp.group_t): if self.is_base_group(k, v): continue log.debug('load group "%s"' % k, self._name) begin = v.get('begin') if begin is None: msg = 'corrupted file structure - group "%s" does not contain key "begin", keys: "%s"' % ( v.name, v.keys()) log.error(msg, self._name) print 'ERROR:', self._name, msg continue end = v.get('end') if end is None: msg = 'corrupted file structure - group "%s" does not contain key "end", keys: "%s"' % ( v.name, v.keys()) log.error(msg, self._name) print 'ERROR:', self._name, msg continue #print 'ZZZ: name, k, v', v.name, v.keys(), v.values() #print "XXX:v['begin'][0], v['end'][0]", v['begin'][0], v['end'][0] o = self.add_range(begin[0], end[0], cmt=False) o.load(v)
def __init__(self, ctype, cmt=None): DCTypeI.__init__(self, ctype, cmt) self._name = self.__class__.__name__ self._dicranges = {} self._ctype = ctype log.debug('In c-tor for ctype: %s' % ctype, self._name)
def _load_pars_dict(self, grp): log.debug('_load_pars_dict for group %s' % grp.name, self._name) self.clear_pars() for k, v in dict(grp).iteritems(): log.debug('par: %s = %s' % (k, str(v[0])), self._name) self.add_par(k, v[0])