def create_recvd_data_group(h5f, title="data analysis and qampy results", description=None, oversampling_dflt=2, attrs=DSP_UNITS, arrays=["data", "symbols", "taps", "bits"], nmodes=2, **kwargs): """ Create the table for saving recovered data and parameters after DSP Parameters ---------- h5f : string or h5filehandle The file to use, if a string create or open new file title: string The title description of the group description: dict or tables.IsDescription (optional) If given use to create the table attrs: dict, optional attributes for the table arrays: list, optional name of arrays referenced in the table nmodes: int, optional number of modes/polarisations **kwargs: keyword arguments passed to create_table/array, it is highly recommended to set expectedrows Returns ------- h5f : h5filehandle Pytables handle to the hdf file """ try: gr = h5f.create_group("/", "analysis", title=title) except AttributeError: h5f = tb.open_file(h5f, "a") gr = h5f.create_group("/", "analysis", title=title) gr_dsp = h5f.create_group(gr, "qampy", title="Signal from DSP") if description is None: dsp_params = { "freq_offset": tb.Float64Col(dflt=np.nan), "freq_offset_N": tb.Int64Col(dflt=0), "phase_est": tb.StringCol(itemsize=20), "N_angles": tb.Float64Col(dflt=np.nan), "ph_est_blocklength": tb.Int64Col(), "stepsize": tb.Float64Col(shape=2), "trsyms": tb.Float64Col(shape=2), "iterations": tb.Int64Col(shape=2), "ntaps": tb.Int64Col(), "method": tb.StringCol(itemsize=20)} description = {"id":tb.Int64Col(), "idx_data": tb.Int64Col(), "idx_symbols": tb.Int64Col(), "idx_bits": tb.Int64Col(), "idx_taps": tb.Int64Col(), "evm": tb.Float64Col(dflt=np.nan, shape=nmodes), "ber":tb.Float64Col(dflt=np.nan, shape=nmodes), "ser":tb.Float64Col(dflt=np.nan, shape=nmodes), "oversampling":tb.Int64Col(dflt=oversampling_dflt)} description.update(dsp_params) t_rec = h5f.create_table(gr_dsp, "signal", description, "signal after DSP", **kwargs) setattr(t_rec.attrs, "arrays", arrays) data_arr = h5f.create_mdvlarray(gr_dsp, "data", tb.ComplexAtom(itemsize=16), "signal after DSP", **kwargs) syms_arr = h5f.create_mdvlarray(gr_dsp, "symbols", tb.ComplexAtom(itemsize=16, dflt=np.nan), "recovered symbols", **kwargs) taps_arr = h5f.create_mdvlarray(gr_dsp, "taps", tb.ComplexAtom(itemsize=16, dflt=np.nan), "qampy taps", **kwargs) bits_arr = h5f.create_mdvlarray(gr_dsp, "bits", tb.BoolAtom(dflt=False), "recovered bits", **kwargs) for k, v in attrs.items(): setattr(t_rec.attrs, k, v) return h5f
def _create_directories(self): """Create appropriate directories for weights histories to be saved. The ID associated with the weights history files is generated using the current time in the form YYMMDD-HHMMSS. A single history file named 'weights_ID.h5' is generated per run of this model. """ time_now = time.strftime('%y, %m, %d, %H, %M, %S').split(', ') date = ''.join(time_now[:3]) self.ID += '%s-%s' % (date, ''.join(time_now[3:])) dir_path = os.getcwd() if dir_path[-1] is not '/': dir_path = dir_path + '/' history_dir = '%shistorical_parameters' % dir_path if not os.path.exists(history_dir): os.makedirs(history_dir) print 'Directory for storing histories has been created.' today_dir = '%s/%s' % (history_dir, date) if not os.path.exists(today_dir): os.makedirs(today_dir) print 'Directory for storing today\'s histories has been created.' weights_file = tables.open_file('%s/weights_%s.h5' % (today_dir, self.ID), mode='w', title='Weights and Biases History') weights_atom = tables.ComplexAtom(itemsize=16, shape=(self.num_visible, self.num_hidden)) visible_bias_atom = tables.ComplexAtom(itemsize=16, shape=(self.num_visible, )) hidden_bias_atom = tables.ComplexAtom(itemsize=16, shape=(self.num_hidden, )) self.weights_history = weights_file.create_earray( weights_file.root, 'weights', title='Historical weights for trial %s' % self.ID, atom=weights_atom, shape=(0, )) self.visible_bias_history = weights_file.create_earray( weights_file.root, 'visible_bias', title='Historical visible biases for trial %s' % self.ID, atom=visible_bias_atom, shape=(0, )) self.hidden_bias_history = weights_file.create_earray( weights_file.root, 'hidden_bias', title='Historical hidden biases for trial %s' % self.ID, atom=hidden_bias_atom, shape=(0, ))
def _calc_ev(self): """ eigenvalues / eigenvectors calculation """ name_eva = 'eva_' + self.digest name_eve = 'eve_' + self.digest csm = self.csm #trigger calculation if (not name_eva in self.h5f.root) or (not name_eve in self.h5f.root): csm_shape = self.csm.shape precisionTuple = _precision(self.precision) eva = empty(csm_shape[0:2], dtype=precisionTuple[0]) eve = empty(csm_shape, dtype=precisionTuple[1]) for i in range(csm_shape[0]): (eva[i], eve[i]) = linalg.eigh(self.csm[i]) atom_eva = precisionTuple[3]() atom_eve = tables.ComplexAtom(precisionTuple[2]) filters = tables.Filters(complevel=5, complib='blosc') ac_eva = self.h5f.create_carray(self.h5f.root, name_eva, atom_eva, \ eva.shape, filters=filters) ac_eve = self.h5f.create_carray(self.h5f.root, name_eve, atom_eve, \ eve.shape, filters=filters) ac_eva[:] = eva ac_eve[:] = eve return (self.h5f.get_node('/', name_eva), \ self.h5f.get_node('/', name_eve))
def create_input_group(h5f, title="input data at transmitter", rolloff_dflt=np.nan, attrs={}, arrays=["symbols", "bits"], **kwargs): """ Create the table for saving the input symbols and bits Parameters ---------- h5f : string or h5filehandle The file to use, if a string create or open new file title: string, optional The title description of the group attrs: dict, optional attributes on the table arrays: list, optional name of arrays referenced in the table **kwargs: keyword arguments passed to create_table/array, it is highly recommended to set expectedrows Returns ------- h5f : h5filehandle Pytables handle to the hdf file """ try: gr = h5f.create_group("/", "input", title=title) except AttributeError: h5f = tb.open_file(h5f, "a") gr = h5f.create_group("/", "input", title=title) # if no shape for input syms or bits is given use scalar t_in = h5f.create_table(gr, "signal", { "id": tb.Int64Col(), "idx_symbols": tb.Int64Col(dflt=0), "idx_bits": tb.Int64Col(dflt=0), "rolloff": tb.Float64Col(dflt=rolloff_dflt) }, title="parameters of input signal", **kwargs) setattr(t_in.attrs, "arrays", arrays) arr_syms = h5f.create_mdvlarray(gr, "symbols", tb.ComplexAtom(itemsize=16, dflt=np.nan), title="sent symbols", **kwargs) arr_bits = h5f.create_mdvlarray(gr, "bits", tb.BoolAtom(), title="sent bits", **kwargs) for k, v in attrs: setattr(t_in.attrs, k, v) return h5f
def create_meas_group(h5f, title="measurement data", description=None, attrs=MEAS_UNITS, arrays=["data"], **kwargs): """ Create the table for saving oscilloscope measurements Parameters ---------- h5f : string or h5filehandle The file to use, if a string create or open new file title: string, optional The title description of the group data_shape: int Number of modes/polarizations description: dict or tables.IsDescription (optional) If given use to create the table arrays: list, optional name of arrays referenced in the table attrs: dict, optional attributes on the table **kwargs: keyword arguments passed to create_table/array, it is highly recommended to set expectedrows Returns ------- h5f : h5filehandle Pytables handle to the hdf file """ try: gr_meas = h5f.create_group("/", "measurements", title=title) except AttributeError: h5f = tb.open_file(h5f, "a") gr_meas = h5f.create_group("/", "measurements", title=title) gr_osc = h5f.create_group(gr_meas, "oscilloscope", title="Data from Realtime oscilloscope") if description is None: description = { "id": tb.Int64Col(), "samplingrate": tb.Float64Col(), "idx_data": tb.Int64Col() } t_meas = h5f.create_table(gr_osc, "signal", description, "sampled signal", **kwargs) setattr(t_meas.attrs, "arrays", arrays) arr = h5f.create_mdvlarray(gr_osc, "data", tb.ComplexAtom(itemsize=16), **kwargs) for k, v in attrs.items(): setattr(t_meas.attrs, k, v) return h5f
def is_cached(self, nodename, group=None): pass def create_compressible_array(self, nodename, shape, precision, group=None): pass if is_tables: precision_to_atom = { 'float32': tables.Float32Atom(), 'complex64': tables.ComplexAtom(8), 'float64': tables.Float64Atom(), 'complex128': tables.ComplexAtom(16), 'bool': tables.BoolAtom(), 'int32': tables.Int32Atom(), 'int16': tables.Int16Atom(), 'int8': tables.Int8Atom(), } class H5FileTables(H5FileBase, tables.File): def create_extendable_array(self, nodename, shape, precision, group=None): if not group: group = self.root
def make_hdf5(hdf5_save_name, label_list, root_directory='.', nfft=1024, nhop=512, fs=22050, seglen=30): if os.path.exists(hdf5_save_name): warnings.warn( 'hdf5 file {} already exists, new file will not be created'.format( hdf5_save_name)) return file_dict = collect_audio(root_directory, label_list) # hdf5 setup hdf5_file = tables.open_file(hdf5_save_name, mode="w") data_node = hdf5_file.create_group(hdf5_file.root, "Data", "Data") data_atom = tables.Float32Atom( ) if theano.config.floatX == 'float32' else tables.Float64Atom() data_atom_complex = tables.ComplexAtom( 8) if theano.config.floatX == 'float32' else tables.ComplexAtom(16) # data nodes hdf5_file.create_earray(data_node, 'X', atom=data_atom_complex, shape=(0, nfft / 2 + 1), title="features") hdf5_file.create_earray(data_node, 'y', atom=data_atom, shape=(0, len(label_list)), title="targets") targets = range(len(label_list)) window = np.hanning(nfft) file_index = {} offset = 0 for target, key in zip(targets, label_list): print 'Processing %s' % key for f in file_dict[key.lower()]: if f.endswith('.wav'): read_fun = audiolab.wavread elif f.endswith('.au'): read_fun = audiolab.auread elif f.endswith('.mp3'): read_fun = read_mp3 # read audio audio_data, fstmp, _ = read_fun(os.path.join(root_directory, f)) # make mono if len(audio_data.shape) != 1: audio_data = np.sum(audio_data, axis=1) / 2. # work with only first seglen seconds audio_data = audio_data[:fstmp * seglen] # resample audio data if fstmp != fs: audio_data = samplerate.resample(audio_data, fs / float(fstmp), 'sinc_best') # compute dft nframes = (len(audio_data) - nfft) // nhop fft_data = np.zeros((nframes, nfft)) for i in xrange(nframes): sup = i * nhop + np.arange(nfft) fft_data[i, :] = audio_data[sup] * window fft_data = np.fft.fft(fft_data) # write dft frames to hdf5 file data_node.X.append(fft_data[:, :nfft / 2 + 1]) # keeping phase too # write target values to hdf5 file one_hot = np.zeros((nframes, len(label_list))) one_hot[:, target] = 1 data_node.y.append(one_hot) # keep file-level info file_index[f] = (offset, nframes, key.lower(), target) offset += nframes hdf5_file.flush() # write file_index and dft parameters to hdf5 file param_node = hdf5_file.create_group(hdf5_file.root, "Param", "Param") param_atom = tables.ObjectAtom() # save dataset metadata hdf5_file.create_vlarray(param_node, 'file_index', atom=param_atom, title='file_index') param_node.file_index.append(file_index) hdf5_file.create_vlarray(param_node, 'file_dict', atom=param_atom, title='file_dict') param_node.file_dict.append(file_dict) hdf5_file.create_vlarray(param_node, 'fft', atom=param_atom, title='fft') param_node.fft.append({'nfft': nfft, 'nhop': nhop, 'window': window}) hdf5_file.create_vlarray(param_node, 'label_list', atom=param_atom, title='label_list') param_node.label_list.append(label_list) hdf5_file.create_vlarray(param_node, 'targets', atom=param_atom, title='targets') param_node.targets.append(targets) hdf5_file.close() print '' # newline
def _get_csm(self): """ Main work is done here: Cross spectral matrix is either loaded from cache file or calculated and then additionally stored into cache. """ # test for dual calibration obj = self.time_data # start with time_data obj while obj: if 'calib' in obj.all_trait_names(): # at original source? if obj.calib and self.calib: if obj.calib.digest == self.calib.digest: self.calib = None # ignore it silently else: raise ValueError("Non-identical dual calibration for "\ "both TimeSamples and PowerSpectra object") obj = None else: try: obj = obj.source # traverse down until original data source except AttributeError: obj = None name = 'csm_' + self.digest H5cache.get_cache(self, self.basename) #print self.basename if not self.cached or not name in self.h5f.root: t = self.time_data wind = self.window_(self.block_size) weight = dot(wind, wind) wind = wind[newaxis, :].swapaxes(0, 1) numfreq = int(self.block_size / 2 + 1) csm_shape = (numfreq, t.numchannels, t.numchannels) csmUpper = zeros(csm_shape, dtype=self.precision) #print "num blocks", self.num_blocks # for backward compatibility if self.calib and self.calib.num_mics > 0: if self.calib.num_mics == t.numchannels: wind = wind * self.calib.data[newaxis, :] else: raise ValueError( "Calibration data not compatible: %i, %i" % \ (self.calib.num_mics, t.numchannels)) bs = self.block_size temp = empty((2 * bs, t.numchannels)) pos = bs posinc = bs / self.overlap_ for data in t.result(bs): ns = data.shape[0] temp[bs:bs + ns] = data while pos + bs <= bs + ns: ft = fft.rfft(temp[int(pos):int(pos + bs)] * wind, None, 0).astype(self.precision) calcCSM( csmUpper, ft ) # only upper triangular part of matrix is calculated (for speed reasons) pos += posinc temp[0:bs] = temp[bs:] pos -= bs # create the full csm matrix via transposingand complex conj. csmLower = csmUpper.conj().transpose(0, 2, 1) [ fill_diagonal(csmLower[cntFreq, :, :], 0) for cntFreq in xrange(csmLower.shape[0]) ] csm = csmLower + csmUpper # onesided spectrum: multiplication by 2.0=sqrt(2)^2 csm = csm * (2.0 / self.block_size / weight / self.num_blocks) if self.cached: precisionTuple = _precision(self.precision) atom = tables.ComplexAtom(precisionTuple[2]) filters = tables.Filters(complevel=5, complib='blosc') ac = self.h5f.create_carray(self.h5f.root, name, atom, csm_shape, filters=filters) ac[:] = csm return ac else: return csm else: return self.h5f.get_node('/', name)
def opensourcefile(k, filename=None, sourcetype=None, overwrite=False): """Open the source term hdf5 file with filename.""" import tables #Set up file for results if not filename or not os.path.isdir(os.path.dirname(filename)): source_logger.info("File or path to file %s does not exist." % filename) date = time.strftime("%Y%m%d%H%M%S") filename = os.path.join(os.getcwd(), "src" + date + ".hf5") source_logger.info("Saving source results in file " + filename) if not sourcetype: raise TypeError( "Need to specify filename and type of source data to store [int(egrand)|(full)term]!" ) if sourcetype in ["int", "term"]: sarrname = "source" + sourcetype if _debug: source_logger.debug("Source array type: " + sarrname) else: raise TypeError("Incorrect source type specified!") #Check if file exists and set write flags depending on overwrite option if os.path.isfile(filename): if overwrite: source_logger.info("File %s exists and will be overwritten." % filename) writeflag = "w" else: source_logger.info("File %s exists and results will be appended." % filename) writeflag = "a" else: writeflag = "w" #Add compression to files and specify good chunkshape filters = tables.Filters(complevel=1, complib=configuration.hdf5complib) #cshape = (10,10,10) #good mix of t, k, q values #Get atom shape for earray atomshape = (0, len(k)) try: if _debug: source_logger.debug("Trying to open source file " + filename) rf = tables.openFile(filename, writeflag, "Source term result") if not "results" in rf.root: if _debug: source_logger.debug("Creating group 'results' in source file.") resgrp = rf.createGroup(rf.root, "results", "Results") else: resgrp = rf.root.results if not sarrname in resgrp: if _debug: source_logger.debug("Creating array '" + sarrname + "' in source file.") sarr = rf.createEArray(resgrp, sarrname, tables.ComplexAtom(itemsize=16), atomshape, filters=filters) karr = rf.createEArray(resgrp, "k", tables.Float64Atom(), (0, ), filters=filters) narr = rf.createEArray(resgrp, "nix", tables.IntAtom(), (0, ), filters=filters) karr.append(k) else: if _debug: source_logger.debug( "Source file and node exist. Testing source node shape...") sarr = rf.getNode(resgrp, sarrname) narr = rf.getNode(resgrp, "nix") if sarr.shape[1:] != atomshape[1:]: raise ValueError("Source node on file is not correct shape!") except IOError: raise return rf, sarr, narr
for s in sets: if not lsts.has_key(s): m, g, v, x = capo.omni.from_npz(sets[s], pols='xx',verbose=True) lsts[s] = m['lsts'] # #chisqs[s] = meta['chisq'][:,CH0:CH0+NCHAN] # for pol in vismdl: # for bl in SEPS: # k = (s,pol,bl) # data[k] = vismdl[pol][bl][:,CH0:CH0+NCHAN] # if bl in CONJ: data[k] = data[k].conj() # data[k] *= bandpass[:,CH0:CH0+NCHAN] # wgts[k] = np.where(np.abs(data[k]) == 0, 0., 1) #lass Vis(IsDescription): Vis = {} for bls in v['xx'].keys(): Vis[bls] = tb.ComplexAtom(itemsize=8) import IPython; IPython.embed() def to_hdf5(m,g,v,x,out_name='test.h5'): h5file = open_file(out_name, mode = "w", title = "Test file") meta = h5file.create_group("/", 'meta', 'Metadata including history, jds, lsts, and various chisq') gains = h5file.create_group("/", 'gains', 'Gains of antennas') vismdl = h5file.create_group("/", 'vismdl', 'Visibilities of baselines given polarization') xtalk = h5file.create_group("/", 'xtalk', 'cross-talk between antennas given polarization') for pol in ['xx']: table = h5file.create_table(root.vismdl, pol, Vis, "Polarization: "+pol)
def msf(el, ns, Hi, order, set_id, wrt_file): st = time.time() # import microstructures microstructure = sio.loadmat('M_%s%s.mat' % (ns, set_id))['M'] microstructure = microstructure.swapaxes(0, 1) tmp = np.zeros([Hi, ns, el**3]) for h in xrange(Hi): tmp[h, ...] = (microstructure == h).astype(int) del microstructure tmp = tmp.swapaxes(0, 1) pre_micr = tmp.reshape([ns, Hi, el, el, el]) del tmp # find H [micr, H] = mf(pre_micr[0, ...], el, Hi, order) del micr # create HDF5 file base = tb.open_file("ref_%s%s.h5" % (ns, set_id), mode="w", title="data for set %s%s" % (ns, set_id)) # create a group one level below root called data group = base.create_group("/", 'msf', 'microstructure functions') # initialize array for the original microstructure base.create_array(group, 'pre_micr', pre_micr, 'the original representation of the microstructure') # close the HDF5 file base.close() # open HDF5 file base = tb.open_file("D_%s%s.h5" % (ns, set_id), mode="w", title="data for set %s%s" % (ns, set_id)) # create a group one level below root for the microstructure function group = base.create_group("/", 'msf', 'microstructure functions') # define the datatype for our array atom = tb.ComplexAtom(16) # initialize an array in the level below group M_all = base.create_carray(group, 'M_all', atom, (ns, H, el**3)) for sn in xrange(ns): # real space microstructure coefficients [micr, H] = mf(pre_micr[sn, ...], el, Hi, order) # take FFT of microstructure coefficients M = np.fft.fftn(micr, axes=[1, 2, 3]).reshape([H, el**3]) del micr M_all[sn, ...] = M del M # close the HDF5 file base.close() msg = "generate real space microstructure and perform FFT:"\ " %s seconds" % np.round((time.time() - st), 3) rr.WP(msg, wrt_file) return H