def create_recvd_data_group(h5f, title="data analysis and qampy results", description=None, oversampling_dflt=2,
attrs=DSP_UNITS, arrays=["data", "symbols", "taps", "bits"], nmodes=2, **kwargs):
"""
Create the table for saving recovered data and parameters after DSP
Parameters
----------
h5f : string or h5filehandle
The file to use, if a string create or open new file
title: string
The title description of the group
description: dict or tables.IsDescription (optional)
If given use to create the table
attrs: dict, optional
attributes for the table
arrays: list, optional
name of arrays referenced in the table
nmodes: int, optional
number of modes/polarisations
**kwargs:
keyword arguments passed to create_table/array, it is highly recommended to set expectedrows
Returns
-------
h5f : h5filehandle
Pytables handle to the hdf file
"""
try:
gr = h5f.create_group("/", "analysis", title=title)
except AttributeError:
h5f = tb.open_file(h5f, "a")
gr = h5f.create_group("/", "analysis", title=title)
gr_dsp = h5f.create_group(gr, "qampy", title="Signal from DSP")
if description is None:
dsp_params = { "freq_offset": tb.Float64Col(dflt=np.nan),
"freq_offset_N": tb.Int64Col(dflt=0), "phase_est": tb.StringCol(itemsize=20),
"N_angles": tb.Float64Col(dflt=np.nan), "ph_est_blocklength": tb.Int64Col(),
"stepsize": tb.Float64Col(shape=2), "trsyms": tb.Float64Col(shape=2),
"iterations": tb.Int64Col(shape=2),
"ntaps": tb.Int64Col(),
"method": tb.StringCol(itemsize=20)}
description = {"id":tb.Int64Col(), "idx_data": tb.Int64Col(), "idx_symbols": tb.Int64Col(),
"idx_bits": tb.Int64Col(), "idx_taps": tb.Int64Col(),
"evm": tb.Float64Col(dflt=np.nan, shape=nmodes), "ber":tb.Float64Col(dflt=np.nan, shape=nmodes),
"ser":tb.Float64Col(dflt=np.nan, shape=nmodes), "oversampling":tb.Int64Col(dflt=oversampling_dflt)}
description.update(dsp_params)
t_rec = h5f.create_table(gr_dsp, "signal", description, "signal after DSP", **kwargs)
setattr(t_rec.attrs, "arrays", arrays)
data_arr = h5f.create_mdvlarray(gr_dsp, "data", tb.ComplexAtom(itemsize=16), "signal after DSP", **kwargs)
syms_arr = h5f.create_mdvlarray(gr_dsp, "symbols", tb.ComplexAtom(itemsize=16, dflt=np.nan), "recovered symbols", **kwargs)
taps_arr = h5f.create_mdvlarray(gr_dsp, "taps", tb.ComplexAtom(itemsize=16, dflt=np.nan), "qampy taps", **kwargs)
bits_arr = h5f.create_mdvlarray(gr_dsp, "bits", tb.BoolAtom(dflt=False), "recovered bits", **kwargs)
for k, v in attrs.items():
setattr(t_rec.attrs, k, v)
return h5f
def _create_directories(self):
"""Create appropriate directories for weights histories to be saved.
The ID associated with the weights history files is generated using
the current time in the form YYMMDD-HHMMSS. A single history file
named 'weights_ID.h5' is generated per run of this model.
"""
time_now = time.strftime('%y, %m, %d, %H, %M, %S').split(', ')
date = ''.join(time_now[:3])
self.ID += '%s-%s' % (date, ''.join(time_now[3:]))
dir_path = os.getcwd()
if dir_path[-1] is not '/':
dir_path = dir_path + '/'
history_dir = '%shistorical_parameters' % dir_path
if not os.path.exists(history_dir):
os.makedirs(history_dir)
print 'Directory for storing histories has been created.'
today_dir = '%s/%s' % (history_dir, date)
if not os.path.exists(today_dir):
os.makedirs(today_dir)
print 'Directory for storing today\'s histories has been created.'
weights_file = tables.open_file('%s/weights_%s.h5' %
(today_dir, self.ID),
mode='w',
title='Weights and Biases History')
weights_atom = tables.ComplexAtom(itemsize=16,
shape=(self.num_visible,
self.num_hidden))
visible_bias_atom = tables.ComplexAtom(itemsize=16,
shape=(self.num_visible, ))
hidden_bias_atom = tables.ComplexAtom(itemsize=16,
shape=(self.num_hidden, ))
self.weights_history = weights_file.create_earray(
weights_file.root,
'weights',
title='Historical weights for trial %s' % self.ID,
atom=weights_atom,
shape=(0, ))
self.visible_bias_history = weights_file.create_earray(
weights_file.root,
'visible_bias',
title='Historical visible biases for trial %s' % self.ID,
atom=visible_bias_atom,
shape=(0, ))
self.hidden_bias_history = weights_file.create_earray(
weights_file.root,
'hidden_bias',
title='Historical hidden biases for trial %s' % self.ID,
atom=hidden_bias_atom,
shape=(0, ))
def _calc_ev(self):
"""
eigenvalues / eigenvectors calculation
"""
name_eva = 'eva_' + self.digest
name_eve = 'eve_' + self.digest
csm = self.csm #trigger calculation
if (not name_eva in self.h5f.root) or (not name_eve in self.h5f.root):
csm_shape = self.csm.shape
precisionTuple = _precision(self.precision)
eva = empty(csm_shape[0:2], dtype=precisionTuple[0])
eve = empty(csm_shape, dtype=precisionTuple[1])
for i in range(csm_shape[0]):
(eva[i], eve[i]) = linalg.eigh(self.csm[i])
atom_eva = precisionTuple[3]()
atom_eve = tables.ComplexAtom(precisionTuple[2])
filters = tables.Filters(complevel=5, complib='blosc')
ac_eva = self.h5f.create_carray(self.h5f.root, name_eva, atom_eva, \
eva.shape, filters=filters)
ac_eve = self.h5f.create_carray(self.h5f.root, name_eve, atom_eve, \
eve.shape, filters=filters)
ac_eva[:] = eva
ac_eve[:] = eve
return (self.h5f.get_node('/', name_eva), \
self.h5f.get_node('/', name_eve))
def create_input_group(h5f,
title="input data at transmitter",
rolloff_dflt=np.nan,
attrs={},
arrays=["symbols", "bits"],
**kwargs):
"""
Create the table for saving the input symbols and bits
Parameters
----------
h5f : string or h5filehandle
The file to use, if a string create or open new file
title: string, optional
The title description of the group
attrs: dict, optional
attributes on the table
arrays: list, optional
name of arrays referenced in the table
**kwargs:
keyword arguments passed to create_table/array, it is highly recommended to set expectedrows
Returns
-------
h5f : h5filehandle
Pytables handle to the hdf file
"""
try:
gr = h5f.create_group("/", "input", title=title)
except AttributeError:
h5f = tb.open_file(h5f, "a")
gr = h5f.create_group("/", "input", title=title)
# if no shape for input syms or bits is given use scalar
t_in = h5f.create_table(gr,
"signal", {
"id": tb.Int64Col(),
"idx_symbols": tb.Int64Col(dflt=0),
"idx_bits": tb.Int64Col(dflt=0),
"rolloff": tb.Float64Col(dflt=rolloff_dflt)
},
title="parameters of input signal",
**kwargs)
setattr(t_in.attrs, "arrays", arrays)
arr_syms = h5f.create_mdvlarray(gr,
"symbols",
tb.ComplexAtom(itemsize=16, dflt=np.nan),
title="sent symbols",
**kwargs)
arr_bits = h5f.create_mdvlarray(gr,
"bits",
tb.BoolAtom(),
title="sent bits",
**kwargs)
for k, v in attrs:
setattr(t_in.attrs, k, v)
return h5f
def create_meas_group(h5f,
title="measurement data",
description=None,
attrs=MEAS_UNITS,
arrays=["data"],
**kwargs):
"""
Create the table for saving oscilloscope measurements
Parameters
----------
h5f : string or h5filehandle
The file to use, if a string create or open new file
title: string, optional
The title description of the group
data_shape: int
Number of modes/polarizations
description: dict or tables.IsDescription (optional)
If given use to create the table
arrays: list, optional
name of arrays referenced in the table
attrs: dict, optional
attributes on the table
**kwargs:
keyword arguments passed to create_table/array, it is highly recommended to set expectedrows
Returns
-------
h5f : h5filehandle
Pytables handle to the hdf file
"""
try:
gr_meas = h5f.create_group("/", "measurements", title=title)
except AttributeError:
h5f = tb.open_file(h5f, "a")
gr_meas = h5f.create_group("/", "measurements", title=title)
gr_osc = h5f.create_group(gr_meas,
"oscilloscope",
title="Data from Realtime oscilloscope")
if description is None:
description = {
"id": tb.Int64Col(),
"samplingrate": tb.Float64Col(),
"idx_data": tb.Int64Col()
}
t_meas = h5f.create_table(gr_osc, "signal", description, "sampled signal",
**kwargs)
setattr(t_meas.attrs, "arrays", arrays)
arr = h5f.create_mdvlarray(gr_osc, "data", tb.ComplexAtom(itemsize=16),
**kwargs)
for k, v in attrs.items():
setattr(t_meas.attrs, k, v)
return h5f
def is_cached(self, nodename, group=None):
pass
def create_compressible_array(self,
nodename,
shape,
precision,
group=None):
pass
if is_tables:
precision_to_atom = {
'float32': tables.Float32Atom(),
'complex64': tables.ComplexAtom(8),
'float64': tables.Float64Atom(),
'complex128': tables.ComplexAtom(16),
'bool': tables.BoolAtom(),
'int32': tables.Int32Atom(),
'int16': tables.Int16Atom(),
'int8': tables.Int8Atom(),
}
class H5FileTables(H5FileBase, tables.File):
def create_extendable_array(self,
nodename,
shape,
precision,
group=None):
if not group: group = self.root
def make_hdf5(hdf5_save_name,
label_list,
root_directory='.',
nfft=1024,
nhop=512,
fs=22050,
seglen=30):
if os.path.exists(hdf5_save_name):
warnings.warn(
'hdf5 file {} already exists, new file will not be created'.format(
hdf5_save_name))
return
file_dict = collect_audio(root_directory, label_list)
# hdf5 setup
hdf5_file = tables.open_file(hdf5_save_name, mode="w")
data_node = hdf5_file.create_group(hdf5_file.root, "Data", "Data")
data_atom = tables.Float32Atom(
) if theano.config.floatX == 'float32' else tables.Float64Atom()
data_atom_complex = tables.ComplexAtom(
8) if theano.config.floatX == 'float32' else tables.ComplexAtom(16)
# data nodes
hdf5_file.create_earray(data_node,
'X',
atom=data_atom_complex,
shape=(0, nfft / 2 + 1),
title="features")
hdf5_file.create_earray(data_node,
'y',
atom=data_atom,
shape=(0, len(label_list)),
title="targets")
targets = range(len(label_list))
window = np.hanning(nfft)
file_index = {}
offset = 0
for target, key in zip(targets, label_list):
print 'Processing %s' % key
for f in file_dict[key.lower()]:
if f.endswith('.wav'):
read_fun = audiolab.wavread
elif f.endswith('.au'):
read_fun = audiolab.auread
elif f.endswith('.mp3'):
read_fun = read_mp3
# read audio
audio_data, fstmp, _ = read_fun(os.path.join(root_directory, f))
# make mono
if len(audio_data.shape) != 1:
audio_data = np.sum(audio_data, axis=1) / 2.
# work with only first seglen seconds
audio_data = audio_data[:fstmp * seglen]
# resample audio data
if fstmp != fs:
audio_data = samplerate.resample(audio_data, fs / float(fstmp),
'sinc_best')
# compute dft
nframes = (len(audio_data) - nfft) // nhop
fft_data = np.zeros((nframes, nfft))
for i in xrange(nframes):
sup = i * nhop + np.arange(nfft)
fft_data[i, :] = audio_data[sup] * window
fft_data = np.fft.fft(fft_data)
# write dft frames to hdf5 file
data_node.X.append(fft_data[:, :nfft / 2 + 1]) # keeping phase too
# write target values to hdf5 file
one_hot = np.zeros((nframes, len(label_list)))
one_hot[:, target] = 1
data_node.y.append(one_hot)
# keep file-level info
file_index[f] = (offset, nframes, key.lower(), target)
offset += nframes
hdf5_file.flush()
# write file_index and dft parameters to hdf5 file
param_node = hdf5_file.create_group(hdf5_file.root, "Param", "Param")
param_atom = tables.ObjectAtom()
# save dataset metadata
hdf5_file.create_vlarray(param_node,
'file_index',
atom=param_atom,
title='file_index')
param_node.file_index.append(file_index)
hdf5_file.create_vlarray(param_node,
'file_dict',
atom=param_atom,
title='file_dict')
param_node.file_dict.append(file_dict)
hdf5_file.create_vlarray(param_node, 'fft', atom=param_atom, title='fft')
param_node.fft.append({'nfft': nfft, 'nhop': nhop, 'window': window})
hdf5_file.create_vlarray(param_node,
'label_list',
atom=param_atom,
title='label_list')
param_node.label_list.append(label_list)
hdf5_file.create_vlarray(param_node,
'targets',
atom=param_atom,
title='targets')
param_node.targets.append(targets)
hdf5_file.close()
print '' # newline
def _get_csm(self):
"""
Main work is done here:
Cross spectral matrix is either loaded from cache file or
calculated and then additionally stored into cache.
"""
# test for dual calibration
obj = self.time_data # start with time_data obj
while obj:
if 'calib' in obj.all_trait_names(): # at original source?
if obj.calib and self.calib:
if obj.calib.digest == self.calib.digest:
self.calib = None # ignore it silently
else:
raise ValueError("Non-identical dual calibration for "\
"both TimeSamples and PowerSpectra object")
obj = None
else:
try:
obj = obj.source # traverse down until original data source
except AttributeError:
obj = None
name = 'csm_' + self.digest
H5cache.get_cache(self, self.basename)
#print self.basename
if not self.cached or not name in self.h5f.root:
t = self.time_data
wind = self.window_(self.block_size)
weight = dot(wind, wind)
wind = wind[newaxis, :].swapaxes(0, 1)
numfreq = int(self.block_size / 2 + 1)
csm_shape = (numfreq, t.numchannels, t.numchannels)
csmUpper = zeros(csm_shape, dtype=self.precision)
#print "num blocks", self.num_blocks
# for backward compatibility
if self.calib and self.calib.num_mics > 0:
if self.calib.num_mics == t.numchannels:
wind = wind * self.calib.data[newaxis, :]
else:
raise ValueError(
"Calibration data not compatible: %i, %i" % \
(self.calib.num_mics, t.numchannels))
bs = self.block_size
temp = empty((2 * bs, t.numchannels))
pos = bs
posinc = bs / self.overlap_
for data in t.result(bs):
ns = data.shape[0]
temp[bs:bs + ns] = data
while pos + bs <= bs + ns:
ft = fft.rfft(temp[int(pos):int(pos + bs)] * wind, None,
0).astype(self.precision)
calcCSM(
csmUpper, ft
) # only upper triangular part of matrix is calculated (for speed reasons)
pos += posinc
temp[0:bs] = temp[bs:]
pos -= bs
# create the full csm matrix via transposingand complex conj.
csmLower = csmUpper.conj().transpose(0, 2, 1)
[
fill_diagonal(csmLower[cntFreq, :, :], 0)
for cntFreq in xrange(csmLower.shape[0])
]
csm = csmLower + csmUpper
# onesided spectrum: multiplication by 2.0=sqrt(2)^2
csm = csm * (2.0 / self.block_size / weight / self.num_blocks)
if self.cached:
precisionTuple = _precision(self.precision)
atom = tables.ComplexAtom(precisionTuple[2])
filters = tables.Filters(complevel=5, complib='blosc')
ac = self.h5f.create_carray(self.h5f.root,
name,
atom,
csm_shape,
filters=filters)
ac[:] = csm
return ac
else:
return csm
else:
return self.h5f.get_node('/', name)
def opensourcefile(k, filename=None, sourcetype=None, overwrite=False):
"""Open the source term hdf5 file with filename."""
import tables
#Set up file for results
if not filename or not os.path.isdir(os.path.dirname(filename)):
source_logger.info("File or path to file %s does not exist." %
filename)
date = time.strftime("%Y%m%d%H%M%S")
filename = os.path.join(os.getcwd(), "src" + date + ".hf5")
source_logger.info("Saving source results in file " + filename)
if not sourcetype:
raise TypeError(
"Need to specify filename and type of source data to store [int(egrand)|(full)term]!"
)
if sourcetype in ["int", "term"]:
sarrname = "source" + sourcetype
if _debug:
source_logger.debug("Source array type: " + sarrname)
else:
raise TypeError("Incorrect source type specified!")
#Check if file exists and set write flags depending on overwrite option
if os.path.isfile(filename):
if overwrite:
source_logger.info("File %s exists and will be overwritten." %
filename)
writeflag = "w"
else:
source_logger.info("File %s exists and results will be appended." %
filename)
writeflag = "a"
else:
writeflag = "w"
#Add compression to files and specify good chunkshape
filters = tables.Filters(complevel=1, complib=configuration.hdf5complib)
#cshape = (10,10,10) #good mix of t, k, q values
#Get atom shape for earray
atomshape = (0, len(k))
try:
if _debug:
source_logger.debug("Trying to open source file " + filename)
rf = tables.openFile(filename, writeflag, "Source term result")
if not "results" in rf.root:
if _debug:
source_logger.debug("Creating group 'results' in source file.")
resgrp = rf.createGroup(rf.root, "results", "Results")
else:
resgrp = rf.root.results
if not sarrname in resgrp:
if _debug:
source_logger.debug("Creating array '" + sarrname +
"' in source file.")
sarr = rf.createEArray(resgrp,
sarrname,
tables.ComplexAtom(itemsize=16),
atomshape,
filters=filters)
karr = rf.createEArray(resgrp,
"k",
tables.Float64Atom(), (0, ),
filters=filters)
narr = rf.createEArray(resgrp,
"nix",
tables.IntAtom(), (0, ),
filters=filters)
karr.append(k)
else:
if _debug:
source_logger.debug(
"Source file and node exist. Testing source node shape...")
sarr = rf.getNode(resgrp, sarrname)
narr = rf.getNode(resgrp, "nix")
if sarr.shape[1:] != atomshape[1:]:
raise ValueError("Source node on file is not correct shape!")
except IOError:
raise
return rf, sarr, narr
for s in sets:
if not lsts.has_key(s):
m, g, v, x = capo.omni.from_npz(sets[s], pols='xx',verbose=True)
lsts[s] = m['lsts']
# #chisqs[s] = meta['chisq'][:,CH0:CH0+NCHAN]
# for pol in vismdl:
# for bl in SEPS:
# k = (s,pol,bl)
# data[k] = vismdl[pol][bl][:,CH0:CH0+NCHAN]
# if bl in CONJ: data[k] = data[k].conj()
# data[k] *= bandpass[:,CH0:CH0+NCHAN]
# wgts[k] = np.where(np.abs(data[k]) == 0, 0., 1)
#lass Vis(IsDescription):
Vis = {}
for bls in v['xx'].keys():
Vis[bls] = tb.ComplexAtom(itemsize=8)
import IPython; IPython.embed()
def to_hdf5(m,g,v,x,out_name='test.h5'):
h5file = open_file(out_name, mode = "w", title = "Test file")
meta = h5file.create_group("/", 'meta', 'Metadata including history, jds, lsts, and various chisq')
gains = h5file.create_group("/", 'gains', 'Gains of antennas')
vismdl = h5file.create_group("/", 'vismdl', 'Visibilities of baselines given polarization')
xtalk = h5file.create_group("/", 'xtalk', 'cross-talk between antennas given polarization')
for pol in ['xx']:
table = h5file.create_table(root.vismdl, pol, Vis, "Polarization: "+pol)
def msf(el, ns, Hi, order, set_id, wrt_file):
st = time.time()
# import microstructures
microstructure = sio.loadmat('M_%s%s.mat' % (ns, set_id))['M']
microstructure = microstructure.swapaxes(0, 1)
tmp = np.zeros([Hi, ns, el**3])
for h in xrange(Hi):
tmp[h, ...] = (microstructure == h).astype(int)
del microstructure
tmp = tmp.swapaxes(0, 1)
pre_micr = tmp.reshape([ns, Hi, el, el, el])
del tmp
# find H
[micr, H] = mf(pre_micr[0, ...], el, Hi, order)
del micr
# create HDF5 file
base = tb.open_file("ref_%s%s.h5" % (ns, set_id),
mode="w",
title="data for set %s%s"
% (ns, set_id))
# create a group one level below root called data
group = base.create_group("/", 'msf', 'microstructure functions')
# initialize array for the original microstructure
base.create_array(group,
'pre_micr',
pre_micr,
'the original representation of the microstructure')
# close the HDF5 file
base.close()
# open HDF5 file
base = tb.open_file("D_%s%s.h5" % (ns, set_id),
mode="w",
title="data for set %s%s"
% (ns, set_id))
# create a group one level below root for the microstructure function
group = base.create_group("/", 'msf', 'microstructure functions')
# define the datatype for our array
atom = tb.ComplexAtom(16)
# initialize an array in the level below group
M_all = base.create_carray(group, 'M_all', atom, (ns, H, el**3))
for sn in xrange(ns):
# real space microstructure coefficients
[micr, H] = mf(pre_micr[sn, ...], el, Hi, order)
# take FFT of microstructure coefficients
M = np.fft.fftn(micr, axes=[1, 2, 3]).reshape([H, el**3])
del micr
M_all[sn, ...] = M
del M
# close the HDF5 file
base.close()
msg = "generate real space microstructure and perform FFT:"\
" %s seconds" % np.round((time.time() - st), 3)
rr.WP(msg, wrt_file)
return H