def calculate_spectrogram(input_array, framesize, hopsize, window_function = numarray.linear_algebra.mlab.hanning, keep_bands_until = 0, axis = 0): """Calculate the spectrogram.""" do_fft = lambda arr: abs(numarray.fft.real_fft(arr, axis = axis))[:keep_bands_until] keep_bands_until = keep_bands_until or int(framesize / 2) input_shape = map(None, numarray.shape(input_array)) window = window_function(framesize) if len(input_shape) > 1: window = transpose(array([list(window)] * input_shape[1])) # print "input_shape:", shape(input_array) zeros_shape = input_shape # this allows for both 1 and 2 dim inputs zeros_shape[0] = framesize / 2 input_array_plus_zeros = numarray.concatenate((numarray.zeros(zeros_shape), input_array, numarray.zeros(zeros_shape))) fft_range = range(0, len(input_array) - framesize, hopsize) fft_array = numarray.zeros(([len(fft_range)] + map(None, numarray.shape(do_fft(window)))), # do_fft(window) is used here because it gives the right shape numarray.Float32) * 1.0 # this *1.0 is necessary! for result_counter, input_counter in enumerate(fft_range): frame = window * input_array_plus_zeros[input_counter : input_counter + framesize] fft_array[result_counter] = 10 * numarray.log10(0.1 + do_fft(frame)) return fft_array
def read_data(filename, hdu = -1, verbose=0):
#####################################################################
# Reads and return a NumPy array containing the data in extension #
# hdu. #
#####################################################################
if verbose: print "Opening ",filename,
p = pcfitsio.fits_open_file(filename,0)
if verbose: print ". Done."
if hdu!=-1:
pcfitsio.fits_movabs_hdu(p,hdu)
if verbose: print "Reading image size.",
n = pcfitsio.fits_read_keys_lng(p,"NAXIS",1,10)
if verbose: print n,"Done."
if n==[] or n==[0]:
pcfitsio.fits_close_file(p)
raise KeyError," Could not read NAXIS values"
if verbose: print "Reading data.",
data = 0
l = []
for i in range(len(n)):
l.append(n[len(n)-i-1])
data = pcfitsio.fits_read_img(p,1,numarray.multiply.reduce(l) ,0.)[0]
if verbose: print "Data read ok.",numarray.shape(data),numarray.multiply.reduce(l)
if verbose: print "Reshape ",l
data=numarray.reshape(data,l)
if verbose: print "Reshape ",l," ok."
if verbose: print "Read",n,"=",numarray.multiply.reduce(n),"elements.",
if verbose: print "Done."
if verbose: print "Closing",filename,
pcfitsio.fits_close_file(p)
if verbose: print ".Done."
return data
def fig3b(t, m, w, p):
""" Plot image of age,metallicity weights -- nearest neighbor interpolation"""
NREP = 50
xi = pylab.linspace(0.1, 15, 512)
yi = pylab.linspace(-2.5, 0.9, 512)
z = numarray.zeros((len(xi), len(yi)),
numarray.Float) # create t, metallicity array
y = numarray.repeat(m, NREP)
dt = t[1] - t[0]
x = numarray.arange(t[0], t[-1] + 2 * dt, dt / NREP)
x = x[0:len(y)]
x = 10.**x / 1.e9
weight = numarray.repeat(w, NREP)
# Find the indices in the array
xindex = numarray.searchsorted(xi, x)
print "shape(x), shape(y), shape(weight)", numarray.shape(
x), numarray.shape(y), numarray.shape(weight)
if p.sigma > 0:
if p.dsigmadlogt == 0.:
for i in range(len(y)):
nstars = weight[i] * normgauss(yi, y[i], p.sigma)
j = xindex[i]
z[j, :] += nstars
if p.dsigmadlogt != 0.:
for i in range(len(y)):
logt0 = numarray.log10(x[0])
logt = numarray.log10(x[i])
sigma = p.sigma + p.dsigmadlogt * (logt - logt0)
nstars = weight[i] * normgauss(yi, y[i], sigma)
j = xindex[i]
z[j, :] += nstars
else:
sigma = 0.01
for i in range(len(y)):
nstars = weight[i] * normgauss(yi, y[i], sigma)
j = xindex[i]
z[j, :] += nstars
# yindex = numarray.searchsorted(yi,y)
# z[xindex,yindex] = z[xindex,yindex] + weight # increment the 2-d array
zz = numarray.transpose(z)
pylab.imshow(zz, extent=[0.1, 15, -2.5, 0.9], aspect='auto')
pylab.xlabel("Age (Gyr)")
pylab.ylabel("[Fe/H]")
return xi, yi, zz
def fig3b(t,m,w,p):
""" Plot image of age,metallicity weights -- nearest neighbor interpolation"""
NREP = 50
xi = pylab.linspace(0.1,15,512)
yi = pylab.linspace(-2.5,0.9,512)
z = numarray.zeros((len(xi),len(yi)),numarray.Float) # create t, metallicity array
y = numarray.repeat(m,NREP)
dt = t[1]-t[0]
x = numarray.arange(t[0],t[-1]+2*dt,dt/NREP)
x = x[0:len(y)]
x = 10.**x/1.e9
weight = numarray.repeat(w,NREP)
# Find the indices in the array
xindex = numarray.searchsorted(xi,x)
print "shape(x), shape(y), shape(weight)", numarray.shape(x),numarray.shape(y), numarray.shape(weight)
if p.sigma > 0:
if p.dsigmadlogt == 0.:
for i in range(len(y)):
nstars = weight[i]*normgauss(yi,y[i],p.sigma)
j = xindex[i]
z[j,:] += nstars
if p.dsigmadlogt != 0.:
for i in range(len(y)):
logt0 = numarray.log10(x[0])
logt = numarray.log10(x[i])
sigma = p.sigma + p.dsigmadlogt*(logt-logt0)
nstars = weight[i]*normgauss(yi,y[i],sigma)
j = xindex[i]
z[j,:] += nstars
else:
sigma = 0.01
for i in range(len(y)):
nstars = weight[i]*normgauss(yi,y[i],sigma)
j = xindex[i]
z[j,:] += nstars
# yindex = numarray.searchsorted(yi,y)
# z[xindex,yindex] = z[xindex,yindex] + weight # increment the 2-d array
zz = numarray.transpose(z)
pylab.imshow(zz,extent=[0.1,15,-2.5,0.9],aspect='auto')
pylab.xlabel("Age (Gyr)")
pylab.ylabel("[Fe/H]")
return xi,yi,zz
def decode_mixture(P, entropy_cutoff):
""" Given P, a (|sequences| x |models|)-matrix where P_{ij} =
P[model j| sequence i] return a list of size (|sequences|)
which contains j, the model which maximizes P[model j| sequence i]
for a sequence, if the entropy of the discrete distribution
{ P[ . | sequence i] } is less than the entropy_cutoff and None else.
"""
nr_seqs = numarray.shape(P)[0]
result = [None] * nr_seqs
for i in xrange(nr_seqs):
e = Entropy(P[i])
#print e, P[i]
if e < entropy_cutoff:
result[i] = int(numarray.argmax(P[i]))
return result
def decode_mixture(P, entropy_cutoff):
""" Given P, a (|sequences| x |models|)-matrix where P_{ij} =
P[model j| sequence i] return a list of size (|sequences|)
which contains j, the model which maximizes P[model j| sequence i]
for a sequence, if the entropy of the discrete distribution
{ P[ . | sequence i] } is less than the entropy_cutoff and None else.
"""
nr_seqs = numarray.shape(P)[0]
result = [None] * nr_seqs
for i in xrange(nr_seqs):
e = Entropy(P[i])
#print e, P[i]
if e < entropy_cutoff:
result[i] = int(numarray.argmax(P[i]))
return result
def fig3e(t,m,w,p):
""" Plot image of log(age),metallicity weights -- nearest neighbor interpolation"""
xi = pylab.linspace(8,10.25,512)
yi = pylab.linspace(-2.5,0.9,512)
z = numarray.zeros((len(xi),len(yi)),numarray.Float) # create t, metallicity array
x = t
y = m
# Find the indices in the array
xindex = numarray.searchsorted(xi,x)
if p.sigma > 0:
for i in range(len(y)):
nstars = w[i]*normgauss(yi,y[i],p.sigma)
print "shape(z),len(nstars)", numarray.shape(z), len(nstars)
z[xindex[i],:] += nstars
else:
yindex = numarray.searchsorted(yi,y)
z[xindex,yindex] = z[xindex,yindex] + weight # increment the 2-d array
zz = numarray.transpose(z)
pylab.imshow(zz,extent=[8,10.25,-2.5,0.9],aspect='auto')
def plot_PCA_residuals(data, D=None, newfig=True, marker='o'):
if D is None:
D = shape(data)[1]
p=doPCA(data, D, D)
spec=zeros((D,1),Float)
for i in range(1,D):
spec[i] = norm(p.d[:i])
res = transpose(sqrt(spec[-1]**2-spec**2)/spec[-1])[0]
print "2-norm of PCA spectrum =", spec[-1]
if newfig:
figure()
style='k'+marker
else:
style=marker
semilogy(res,style)
## title('PCA residuals')
xlabel(r'$\rm{Dimension}$',fontsize=20)
ylabel(r'$\rm{PCA \ residual}$',fontsize=20)
return p, res
def plot_PCA_residuals(data, D=None, newfig=True, marker='o'):
if D is None:
D = shape(data)[1]
p = doPCA(data, D, D)
spec = zeros((D, 1), Float)
for i in range(1, D):
spec[i] = norm(p.d[:i])
res = transpose(sqrt(spec[-1]**2 - spec**2) / spec[-1])[0]
print "2-norm of PCA spectrum =", spec[-1]
if newfig:
figure()
style = 'k' + marker
else:
style = marker
semilogy(res, style)
## title('PCA residuals')
xlabel(r'$\rm{Dimension}$', fontsize=20)
ylabel(r'$\rm{PCA \ residual}$', fontsize=20)
return p, res
def fig3e(t, m, w, p):
""" Plot image of log(age),metallicity weights -- nearest neighbor interpolation"""
xi = pylab.linspace(8, 10.25, 512)
yi = pylab.linspace(-2.5, 0.9, 512)
z = numarray.zeros((len(xi), len(yi)),
numarray.Float) # create t, metallicity array
x = t
y = m
# Find the indices in the array
xindex = numarray.searchsorted(xi, x)
if p.sigma > 0:
for i in range(len(y)):
nstars = w[i] * normgauss(yi, y[i], p.sigma)
print "shape(z),len(nstars)", numarray.shape(z), len(nstars)
z[xindex[i], :] += nstars
else:
yindex = numarray.searchsorted(yi, y)
z[xindex,
yindex] = z[xindex, yindex] + weight # increment the 2-d array
zz = numarray.transpose(z)
pylab.imshow(zz, extent=[8, 10.25, -2.5, 0.9], aspect='auto')
def _rc(self, a):
if len(shape(a)) == 0:
return a
else:
return Matrix(a)
def _rc(self, a):
if len(shape(a)) == 0:
return a
else:
return Matrix(a)
def write_to_hdf_old_style(self, hdffile, where, name, title, complib=None, complevel=None):
accu_group=hdffile.createGroup(where=where,name=name,title=title)
accu_group._v_attrs.damaris_type="Accumulation"
if self.contains_data():
self.lock.acquire()
try:
# save time stamps
if self.time_period is not None and len(self.time_period)>0:
accu_group._v_attrs.earliest_time="%04d%02d%02d %02d:%02d:%02d.%03d"%(self.time_period[0].year,
self.time_period[0].month,
self.time_period[0].day,
self.time_period[0].hour,
self.time_period[0].minute,
self.time_period[0].second,
self.time_period[0].microsecond/1000)
accu_group._v_attrs.oldest_time="%04d%02d%02d %02d:%02d:%02d.%03d"%(self.time_period[1].year,
self.time_period[1].month,
self.time_period[1].day,
self.time_period[1].hour,
self.time_period[1].minute,
self.time_period[1].second,
self.time_period[1].microsecond/1000)
if self.common_descriptions is not None:
for (key,value) in self.common_descriptions.iteritems():
accu_group._v_attrs.__setattr__("description_"+key,str(value))
# save channel data
for channel_no in xrange(len(self.y)):
y_mean=self.get_ydata(channel_no)
y_sigma=self.get_yerr(channel_no)
for index_no in xrange(len(self.index)):
index=self.index[index_no]
# set time data
timedata=numarray.array(type = numarray.Float64,
shape = (2*(index[1]-index[0]+1),))
timedata[0::2]=y_mean[index[0]:index[1]+1]
if len(y_sigma):
timedata[1::2]=y_sigma[index[0]:index[1]+1]
else:
timedata[1::2]=numarray.zeros(type = numarray.Float64,
shape = ((index[1]-index[0]+1),))
timedata.setshape((index[1]-index[0]+1,2))
time_slice_data=None
if complib is not None:
if complevel is None:
complevel=9
chunkshape = numarray.shape(timedata)
if len(chunkshape) <= 1:
chunkshape = (min(chunkshape[0],1024*8),)
else:
chunkshape = (min(chunkshape[0],1024*8), chunkshape[1])
time_slice_data=hdffile.createCArray(accu_group,
name="idx%04d_ch%04d"%(index_no,channel_no),
shape=timedata.getshape(),
atom=tables.Float64Atom(shape=chunkshape,
flavor="numarray"),
filters=tables.Filters(complevel=complevel,
complib=complib,
shuffle=1),
title="Index %d, Channel %d"%(index_no,channel_no))
time_slice_data[:]=timedata
else:
time_slice_data=hdffile.createArray(accu_group,
name="idx%04d_ch%04d"%(index_no,channel_no),
object=timedata,
title="Index %d, Channel %d"%(index_no,channel_no))
timedata=None
# set attributes
time_slice_data._f_setAttr("index",numarray.array(index_no, type=numarray.Int32))
time_slice_data._f_setAttr("channel",numarray.array(channel_no, type=numarray.Int32))
time_slice_data._f_setAttr("number",numarray.array(self.n, type=numarray.Int64))
time_slice_data._f_setAttr("dwelltime",numarray.array(1.0/self.sampling_rate,
type=numarray.Float64))
time_slice_data._f_setAttr("start_time",numarray.array(1.0/self.sampling_rate*index[0],
type=numarray.Float64))
finally:
time_slice_data=None
accu_group=None
self.lock.release()
def whiten(data):
wdata=zeros(shape(data),Float)
for d in range(shape(data)[1]):
x = data[:,d]
wdata[:,d] = (x-mean(x))/std(x)
return wdata
def write_data(filename, data, bitpix = -1, hdu = -1, append=0, verbose=0):
#############################################################################
# Write an array, or a list, to the file filename #
# The type of the array determines the value of bitpix, which can be #
# overwritten if necessary. Data will be written to the extension #
# indicated by the parameter hdu=. If that extension does not exist but the #
# parameter appen= is set to 1, then a new hdu is appended to the data #
# Setting verbose=1 turns on the verbose mode #
#############################################################################
import posixpath
if type(filename) != type(" "): raise InvalidFilename
try:
data = numarray.asarray(data)
# data = numarray.transpose(data)
naxes = numarray.asarray(numarray.shape(data)).tolist()
l = []
for i in range(len(naxes)):
l.append(naxes[len(naxes)-i-1])
naxes=l
except:
raise InvalidData
if bitpix == -1:
if data.typecode()=='l':
bitpix = 32
if data.typecode()=='d':
bitpix = -32
if not (bitpix in [-32,8,16,32]):
print "Invalid BITPIX: ",bitpix
return
ok = 0
new = 1
if (posixpath.isfile(filename)):
p = pcfitsio.fits_open_file(filename,1)
ok = 1
new = 0
if hdu != -1:
try:
if verbose: print "Trying to move to extension #"+`hdu`
pcfitsio.fits_movabs_hdu(p,hdu)
append = 0
ok = 1
except:
if verbose: print "Failed to move to extension #"+`hdu`
if append:
# Create a new image extension
if verbose: print "Appending an image extension.",
pcfitsio.fits_create_img(p,bitpix,naxes)
if verbose: print "Done.",
# Find out the number of extensions
# hdunum = pcfitsio.fits_get_num_hdus(p)
# if verbose: print "Total #hdu:",hdunum
# pcfitsio.fits_movabs_hdu(p,hdunum)
ok = 1
else:
if verbose: print "File not found."
if verbose: print "Creating:",filename,
p = pcfitsio.fits_create_file(filename)
new = 1
if verbose: print "Done."
if verbose: print "Appending an image extension.",
pcfitsio.fits_create_img(p,bitpix,naxes)
pcfitsio.fits_close_file(p)
p = pcfitsio.fits_open_file(filename,1)
if verbose: print "Done.",
# Find out the number of extensions
hdunum = pcfitsio.fits_get_num_hdus(p)
if verbose: print "Total #hdu:",hdunum
pcfitsio.fits_movabs_hdu(p,hdunum)
ok = 1
# Write the data to the file which is now opened/created
if ok:
if new == 0:
if verbose: print "Resizing img ext..",
pcfitsio.fits_resize_img(p,bitpix,naxes)
if verbose: print "Writing data.",naxes,
if verbose: print numarray.multiply.reduce(naxes)
pcfitsio.fits_write_img(p,1,numarray.multiply.reduce(naxes),data)
if verbose: print "Done."
if verbose: print "Closing",filename,
pcfitsio.fits_close_file(p)
if verbose: print "Done."
def write_to_hdf_old_style(self, hdffile, where, name, title, complib=None, complevel=None):
accu_group=hdffile.createGroup(where=where,name=name,title=title)
accu_group._v_attrs.damaris_type="ADC_result"
if self.contains_data():
self.lock.acquire()
try:
# save time stamps
if "job_date" in dir(self) and self.job_date is not None:
accu_group._v_attrs.time="%04d%02d%02d %02d:%02d:%02d.%03d"%(self.job_date.year,
self.job_date.month,
self.job_date.day,
self.job_date.hour,
self.job_date.minute,
self.job_date.second,
self.job_date.microsecond/1000)
if self.description is not None:
for (key,value) in self.description.iteritems():
accu_group._v_attrs.__setattr__("description_"+key,str(value))
# save channel data
for channel_no in xrange(len(self.y)):
y_mean=self.get_ydata(channel_no)
for index_no in xrange(len(self.index)):
index=self.index[index_no]
# set time data
timedata=numarray.array(y_mean[index[0]:index[1]+1],
type = numarray.Int32)
time_slice_data=None
if complib is not None:
if complevel is None:
complevel=9
chunkshape = numarray.shape(timedata)
if len(chunkshape) <= 1:
chunkshape = (min(chunkshape[0],1024*8),)
else:
chunkshape = (min(chunkshape[0],1024*8), chunkshape[1])
time_slice_data=hdffile.createCArray(accu_group,
name="idx%04d_ch%04d"%(index_no,channel_no),
shape=timedata.getshape(),
atom=tables.Int32Atom(shape=chunkshape,
flavor="numarray"),
filters=tables.Filters(complevel=complevel,
complib=complib),
title="Index %d, Channel %d"%(index_no,channel_no))
time_slice_data[:]=timedata
else:
time_slice_data=hdffile.createArray(accu_group,
name="idx%04d_ch%04d"%(index_no,channel_no),
object=timedata,
title="Index %d, Channel %d"%(index_no,channel_no))
timedata=None
# set attributes
time_slice_data._f_setAttr("index",numarray.array(index_no, type=numarray.Int32))
time_slice_data._f_setAttr("channel",numarray.array(channel_no, type=numarray.Int32))
time_slice_data._f_setAttr("dwelltime",numarray.array(1.0/self.sampling_rate,
type=numarray.Float64))
time_slice_data._f_setAttr("start_time",numarray.array(1.0/self.sampling_rate*index[0],
type=numarray.Float64))
finally:
time_slice_data=None
accu_group=None
self.lock.release()
def whiten(data):
wdata = zeros(shape(data), Float)
for d in range(shape(data)[1]):
x = data[:, d]
wdata[:, d] = (x - mean(x)) / std(x)
return wdata
def write_to_hdf_new_style(self, hdffile, where, name, title, complib=None, complevel=None):
accu_group=hdffile.createGroup(where=where,name=name,title=title)
accu_group._v_attrs.damaris_type="ADC_result"
if self.contains_data():
self.lock.acquire()
try:
# save time stamps
if "job_date" in dir(self) and self.job_date is not None:
accu_group._v_attrs.time="%04d%02d%02d %02d:%02d:%02d.%03d"%(self.job_date.year,
self.job_date.month,
self.job_date.day,
self.job_date.hour,
self.job_date.minute,
self.job_date.second,
self.job_date.microsecond/1000)
if self.description is not None:
for (key,value) in self.description.iteritems():
accu_group._v_attrs.__setattr__("description_"+key,str(value))
# save interval information
filter=None
if complib is not None:
if complevel is None:
complevel=9
filter=tables.Filters(complevel=complevel,complib=complib,shuffle=1)
index_table=hdffile.createTable(where=accu_group,
name="indices",
description={"start": tables.UInt64Col(),
"length": tables.UInt64Col(),
"start_time": tables.Float64Col(),
"dwelltime": tables.Float64Col()},
title="indices of adc data intervals",
filters=filter,
expectedrows=len(self.index))
# save channel data
new_row=index_table.row
for i in xrange(len(self.index)):
new_row["start"]=self.index[i][0]
new_row["dwelltime"]=1.0/self.sampling_rate
new_row["start_time"]=1.0/self.sampling_rate*self.index[i][0]
new_row["length"]=self.index[i][1]-self.index[i][0]+1
new_row.append()
index_table.flush()
new_row=None
index_table=None
# prepare saving data
channel_no=len(self.y)
timedata=numarray.array(shape = (len(self.y[0]),channel_no),
type = numarray.Int32)
for ch in xrange(channel_no):
timedata[:,ch]=self.get_ydata(ch)
# save data
time_slice_data=None
if filter is not None:
chunkshape = numarray.shape(timedata)
if len(chunkshape) <= 1:
chunkshape = (min(chunkshape[0],1024*8),)
else:
chunkshape = (min(chunkshape[0],1024*8), chunkshape[1])
time_slice_data=hdffile.createCArray(accu_group,
name="adc_data",
shape=timedata.getshape(),
atom=tables.Int32Atom(shape=chunkshape,
flavor="numarray"),
filters=filter,
title="adc data")
time_slice_data[:]=timedata
else:
time_slice_data=hdffile.createArray(accu_group,
name="adc_data",
object=timedata,
title="adc data")
finally:
timedata=None
time_slice_data=None
accu_group=None
self.lock.release()
