def test_readwrite_hd5_file(self): global test_file_large hdf5file = test_file_large[:-4]+'.hdf5' daq = Daq() daq.read_hd5(os.path.join('data', hdf5file)) do = daq.dynobjs.values()[0] do.write_hd5(filename='./tmp/dynobj_test.hdf5') do2 = DynObj() do2.read_hd5('./tmp/dynobj_test.hdf5') assert_dynobjs_equal(self, do, do2)
def _process_dynobjs(self): """ figure out what rows and frames relate to particular dynamic objects so that they can be unpacked by DynObj.process """ if 'SCC_DynObj_CvedId'not in self: msg = "Need 'SCC_DynObj*' to build dynobjs (did not build dynobjs)" warnings.warn(msg, RuntimeWarning) return None frame = self.frame cvedid_array = self['SCC_DynObj_CvedId'] # identify the cars in the datafile and # identify what frames they are defined frame_indiceses = OrderedDict() row_indiceses = OrderedDict() # loop through frames, skipping first frame for i in xrange(1, len(frame.frame)): cveds_in_frame = cvedid_array[:,i].flatten() for j, cved in enumerate(cveds_in_frame): if cved != 0: try: # if cved not in dict it will raise a # key error frame_indiceses[cved].append(i) row_indiceses[cved].append(j) except: frame_indiceses[cved] = array('i', [i]) row_indiceses[cved] = array('i', [j]) self.dynobjs = OrderedDict() for cvedId in frame_indiceses.keys(): do = DynObj() do.process(cvedId, np.array(frame_indiceses[cvedId]), np.array(row_indiceses[cvedId]), self) self.dynobjs[do.name] = do del frame_indiceses del row_indiceses
def _process_dynobjs(self): """ figure out what rows and frames relate to particular dynamic objects so that they can be unpacked by DynObj.process """ if 'SCC_DynObj_CvedId' not in self: msg = "Need 'SCC_DynObj*' to build dynobjs (did not build dynobjs)" warnings.warn(msg, RuntimeWarning) return None frame = self.frame cvedid_array = self['SCC_DynObj_CvedId'] # identify the cars in the datafile and # identify what frames they are defined frame_indiceses = OrderedDict() row_indiceses = OrderedDict() # loop through frames, skipping first frame for i in xrange(1, len(frame.frame)): cveds_in_frame = cvedid_array[:, i].flatten() for j, cved in enumerate(cveds_in_frame): if cved != 0: try: # if cved not in dict it will raise a # key error frame_indiceses[cved].append(i) row_indiceses[cved].append(j) except: frame_indiceses[cved] = array('i', [i]) row_indiceses[cved] = array('i', [j]) self.dynobjs = OrderedDict() for cvedId in frame_indiceses.keys(): do = DynObj() do.process(cvedId, np.array(frame_indiceses[cvedId]), np.array(row_indiceses[cvedId]), self) self.dynobjs[do.name] = do del frame_indiceses del row_indiceses
def read_hd5(self, filename, f0=None, fend=None): """ read_hd5(filename[, f0=None][, fend=None]) writes Daq object to HDF5 container f0 and fend specify frame range to read. Will export daq.dynobjs to HDF5 container Parameters ---------- filename : string file to read f0 : None or int None -> read from beginning of file int -> read from this frame fend : None or int None -> read to end of file int -> read to this frame Return ------ None """ if filename.endswith('daq'): # Normal Python mantra usually goes against this sort of # of checking but when you make this error the traceback # is somewhat obtuse. It can be difficult to tell whether # it is in the wrong format or something else is wrong. raise ValueError('You are trying to open a .daq as .hd5') root = h5py.File(filename, 'r') # info self.info = \ Info(run = root['info'].attrs['run'], runinst = root['info'].attrs['runinst'], title = root['info'].attrs['title'], numentries = root['info'].attrs['numentries'], frequency = root['info'].attrs['frequency'], date = root['info'].attrs['date'], magic = root['info'].attrs['magic'], subject = root['info'].attrs['subject'], filename = root['info'].attrs['filename']) # header # The [:] unpacks the data from a h5py.dataset.Dataset # object to a numpy.ndarray object try: _header = \ Header(id = root['header/id'][:], numvalues = root['header/numvalues'][:], name = root['header/name'][:], units = root['header/units'][:], rate = root['header/rate'][:], type = root['header/type'][:], nptype = root['header/nptype'][:], varrateflag = root['header/varrateflag'][:], bytes = root['header/bytes'][:]) except: _header = \ Header(id = array('i'), numvalues = array('i'), name = [], units = [], rate = array('i'), type = array('c'), nptype = [], varrateflag = array('i'), bytes = array('i')) # Find the indices cooresponding to the first and last # frames requested. We can use these indices to # slice out all the non-CSSDC elements. # # For the CSSDC elements we will have to find the # appropriate indices as we go. i0, iend = None, None try: all_frames = root['frame/frame'][:] except: all_frames = None if all_frames is not None: if f0 is not None or fend is not None: if f0 is not None: i0 = _searchsorted(all_frames, f0) if fend is not None: iend = _searchsorted(all_frames, fend) if iend < len(all_frames): iend += 1 indx = slice(i0,iend) # frame try: self.frame = \ Frame(code = root['frame/code'][indx], frame = root['frame/frame'][indx], count = root['frame/count'][indx]) except: self.frame = \ Frame(code = array('i'), frame = array('i'), count = array('i')) # elemlist try: # Fails if slice is zero-length self.elemlist = root['elemlist'][:] except: self.elemlist = None # data # Procedure is similar to read_daq. Data is unpacked # to tmpdata dict and then the Elements are instantiated. _elemid_lookup = dict(zip(_header.name, _header.id)) tmpdata = {} for k, v in root['data'].iteritems(): if self.elemlist is not None: if not any(fnmatch(k, wc) for wc in self.elemlist): continue i = _elemid_lookup[k.replace('_Frames','')] if _header.rate[i] == 1: tmpdata[k] = v[:,i0:iend] else: #CSSDC measure if len(v.shape) == 1: v = np.array(v, ndmin=2) # _Frames # Need to find indices _i0 = 0 _iend= v.shape[1] if f0 is not None or fend is not None: _name = k.replace('_Frames','') _all_frames = root['data/%s_Frames'%_name][:].flatten() if f0 is not None: _i0 = _searchsorted(_all_frames, f0) if fend is not None: _iend = _searchsorted(_all_frames, fend) if _iend < len(_all_frames): _iend += 1 # Now we can slice the data tmpdata[k] = v[:,_i0:_iend] # hdf5 doesn't have a None type (or atleast, I don't know how # to use it) so None is stored as an empty string in the hdf5 file self.f0 = (root.attrs['f0'], None)[root.attrs['f0'] == ''] self.fend = (root.attrs['fend'], None)[root.attrs['fend'] == ''] self.cursor = root.attrs['cursor'] # read dynobjs self.dynobjs = OrderedDict() for (name, dynobj) in root['dynobjs'].iteritems(): do = DynObj() do.read_hd5(root=dynobj) self.dynobjs[name] = do # read etc dict self.etc = {} for (name, obj) in root['etc'].attrs.iteritems(): self.etc[name] = _literal_eval(obj) root.close() # cast as Element objects # 'varrateflag' variables remain lists of lists # # There are obvious more compact ways to write this but I'm # paranoid about reference counting and garbage collection not # functioning properly for name, i, rate in zip(_header.name, _header.id, _header.rate): if rate != 1: self[name] = \ Element(tmpdata[name], tmpdata[name+'_Frames'].flatten(), rate=_header.rate[i], name=_header.name[i], dtype=_header.type[i], varrateflag=_header.varrateflag[i], elemid=_header.id[i], units=_header.units[i]) del tmpdata[name+'_Frames'] else: self[name] = \ Element(tmpdata[name], self.frame.frame[:], rate=_header.rate[i], name=_header.name[i], dtype=_header.type[i], varrateflag=_header.varrateflag[i], elemid=_header.id[i], units=_header.units[i]) # delete tmpdata arrays as we go to save memory del tmpdata[name] del _header
def read_hd5(self, filename, f0=None, fend=None): """ read_hd5(filename[, f0=None][, fend=None]) writes Daq object to HDF5 container f0 and fend specify frame range to read. Will export daq.dynobjs to HDF5 container Parameters ---------- filename : string file to read f0 : None or int None -> read from beginning of file int -> read from this frame fend : None or int None -> read to end of file int -> read to this frame Return ------ None """ if filename.endswith('daq'): # Normal Python mantra usually goes against this sort of # of checking but when you make this error the traceback # is somewhat obtuse. It can be difficult to tell whether # it is in the wrong format or something else is wrong. raise ValueError('You are trying to open a .daq as .hd5') root = h5py.File(filename, 'r') # info self.info = \ Info(run = root['info'].attrs['run'], runinst = root['info'].attrs['runinst'], title = root['info'].attrs['title'], numentries = root['info'].attrs['numentries'], frequency = root['info'].attrs['frequency'], date = root['info'].attrs['date'], magic = root['info'].attrs['magic'], subject = root['info'].attrs['subject'], filename = root['info'].attrs['filename']) # header # The [:] unpacks the data from a h5py.dataset.Dataset # object to a numpy.ndarray object try: _header = \ Header(id = root['header/id'][:], numvalues = root['header/numvalues'][:], name = root['header/name'][:], units = root['header/units'][:], rate = root['header/rate'][:], type = root['header/type'][:], nptype = root['header/nptype'][:], varrateflag = root['header/varrateflag'][:], bytes = root['header/bytes'][:]) except: _header = \ Header(id = array('i'), numvalues = array('i'), name = [], units = [], rate = array('i'), type = array('c'), nptype = [], varrateflag = array('i'), bytes = array('i')) # Find the indices cooresponding to the first and last # frames requested. We can use these indices to # slice out all the non-CSSDC elements. # # For the CSSDC elements we will have to find the # appropriate indices as we go. i0, iend = None, None try: all_frames = root['frame/frame'][:] except: all_frames = None if all_frames is not None: if f0 is not None or fend is not None: if f0 is not None: i0 = _searchsorted(all_frames, f0) if fend is not None: iend = _searchsorted(all_frames, fend) if iend < len(all_frames): iend += 1 indx = slice(i0, iend) # frame try: self.frame = \ Frame(code = root['frame/code'][indx], frame = root['frame/frame'][indx], count = root['frame/count'][indx]) except: self.frame = \ Frame(code = array('i'), frame = array('i'), count = array('i')) # elemlist try: # Fails if slice is zero-length self.elemlist = root['elemlist'][:] except: self.elemlist = None # data # Procedure is similar to read_daq. Data is unpacked # to tmpdata dict and then the Elements are instantiated. _elemid_lookup = dict(zip(_header.name, _header.id)) tmpdata = {} for k, v in root['data'].iteritems(): if self.elemlist is not None: if not any(fnmatch(k, wc) for wc in self.elemlist): continue i = _elemid_lookup[k.replace('_Frames', '')] if _header.rate[i] == 1: tmpdata[k] = v[:, i0:iend] else: #CSSDC measure if len(v.shape) == 1: v = np.array(v, ndmin=2) # _Frames # Need to find indices _i0 = 0 _iend = v.shape[1] if f0 is not None or fend is not None: _name = k.replace('_Frames', '') _all_frames = root['data/%s_Frames' % _name][:].flatten() if f0 is not None: _i0 = _searchsorted(_all_frames, f0) if fend is not None: _iend = _searchsorted(_all_frames, fend) if _iend < len(_all_frames): _iend += 1 # Now we can slice the data tmpdata[k] = v[:, _i0:_iend] # hdf5 doesn't have a None type (or atleast, I don't know how # to use it) so None is stored as an empty string in the hdf5 file self.f0 = (root.attrs['f0'], None)[root.attrs['f0'] == ''] self.fend = (root.attrs['fend'], None)[root.attrs['fend'] == ''] self.cursor = root.attrs['cursor'] # read dynobjs self.dynobjs = OrderedDict() for (name, dynobj) in root['dynobjs'].iteritems(): do = DynObj() do.read_hd5(root=dynobj) self.dynobjs[name] = do # read etc dict self.etc = {} for (name, obj) in root['etc'].attrs.iteritems(): self.etc[name] = _literal_eval(obj) root.close() # cast as Element objects # 'varrateflag' variables remain lists of lists # # There are obvious more compact ways to write this but I'm # paranoid about reference counting and garbage collection not # functioning properly for name, i, rate in zip(_header.name, _header.id, _header.rate): if rate != 1: self[name] = \ Element(tmpdata[name], tmpdata[name+'_Frames'].flatten(), rate=_header.rate[i], name=_header.name[i], dtype=_header.type[i], varrateflag=_header.varrateflag[i], elemid=_header.id[i], units=_header.units[i]) del tmpdata[name + '_Frames'] else: self[name] = \ Element(tmpdata[name], self.frame.frame[:], rate=_header.rate[i], name=_header.name[i], dtype=_header.type[i], varrateflag=_header.varrateflag[i], elemid=_header.id[i], units=_header.units[i]) # delete tmpdata arrays as we go to save memory del tmpdata[name] del _header