def sTOsAffine(infile, outfile, locale, globale):
i = 1
global LL
global GG
while i < len(sys.argv):
arg = sys.argv[i]
if infile is None:
infile = arg
elif outfile is None:
outfile = arg
elif locale is None:
locale = arg
elif globale is None:
globale = arg
# elif layer_name is None:
# layer_name = arg
else:
Usage()
i = i + 1
if outfile is None:
Usage()
in_ds = ogr.Open(str(infile), update=0)
if layer_name is not None:
in_layer = in_ds.GetLayerByName(layer_name)
else:
in_layer = in_ds.GetLayer(0)
in_defn = in_layer.GetLayerDefn()
LL = io.read_array(str(locale))
GG = io.read_array(str(globale))
shp_driver = ogr.GetDriverByName('ESRI Shapefile')
shp_driver.DeleteDataSource(str(outfile))
shp_ds = shp_driver.CreateDataSource(str(outfile))
shp_layer = shp_ds.CreateLayer(in_defn.GetName(),
geom_type=in_defn.GetGeomType(),
srs=in_layer.GetSpatialRef())
in_field_count = in_defn.GetFieldCount()
for fld_index in range(in_field_count):
src_fd = in_defn.GetFieldDefn(fld_index)
fd = ogr.FieldDefn(src_fd.GetName(), src_fd.GetType())
fd.SetWidth(src_fd.GetWidth())
fd.SetPrecision(src_fd.GetPrecision())
shp_layer.CreateField(fd)
in_feat = in_layer.GetNextFeature()
while in_feat is not None:
geom = in_feat.GetGeometryRef().Clone()
geom = WalkAndTransformAff(geom)
out_feat = ogr.Feature(feature_def=shp_layer.GetLayerDefn())
out_feat.SetFrom(in_feat)
out_feat.SetGeometryDirectly(geom)
shp_layer.CreateFeature(out_feat)
out_feat.Destroy()
in_feat.Destroy()
in_feat = in_layer.GetNextFeature()
shp_ds.Destroy()
in_ds.Destroy()
def sTOsAffine(infile,outfile,locale,globale): i = 1 global LL global GG while i < len(sys.argv): arg = sys.argv[i] if infile is None: infile = arg elif outfile is None: outfile = arg elif locale is None: locale = arg elif globale is None: globale = arg # elif layer_name is None: # layer_name = arg else: Usage() i = i + 1 if outfile is None: Usage() in_ds = ogr.Open( str(infile), update = 0 ) if layer_name is not None: in_layer = in_ds.GetLayerByName( layer_name ) else: in_layer = in_ds.GetLayer( 0 ) in_defn = in_layer.GetLayerDefn() LL = io.read_array(str(locale)) GG = io.read_array(str(globale)) shp_driver = ogr.GetDriverByName( 'ESRI Shapefile' ) shp_driver.DeleteDataSource( str(outfile) ) shp_ds = shp_driver.CreateDataSource( str(outfile) ) shp_layer = shp_ds.CreateLayer( in_defn.GetName(),geom_type = in_defn.GetGeomType(),srs = in_layer.GetSpatialRef() ) in_field_count = in_defn.GetFieldCount() for fld_index in range(in_field_count): src_fd = in_defn.GetFieldDefn( fld_index ) fd = ogr.FieldDefn( src_fd.GetName(), src_fd.GetType() ) fd.SetWidth( src_fd.GetWidth() ) fd.SetPrecision( src_fd.GetPrecision() ) shp_layer.CreateField( fd ) in_feat = in_layer.GetNextFeature() while in_feat is not None: geom = in_feat.GetGeometryRef().Clone() geom = WalkAndTransformAff( geom ) out_feat = ogr.Feature( feature_def = shp_layer.GetLayerDefn() ) out_feat.SetFrom( in_feat ) out_feat.SetGeometryDirectly( geom ) shp_layer.CreateFeature( out_feat ) out_feat.Destroy() in_feat.Destroy() in_feat = in_layer.GetNextFeature() shp_ds.Destroy() in_ds.Destroy()
def main():
# First example, read first and second column from ascii file. Skip first
# line of header.
# Note use of (1,-1) in lines to skip first line and then read to end of file
# Note use of (0,) in columns to pick first column, since its a tuple need trailing comma
x=read_array("test.txt",lines=(1,-1), columns=(0,))
y=read_array("test.txt",lines=(1,-1), columns=(1,))
#Second example, read the file into a single arry
z=read_array("test.txt",lines=(1,-1), columns=(0,2))
# Plot the data
plot(x,y,'r--',z[:,0],z[:,1])
show()
def main():
# First example, read first and second column from ascii file. Skip first
# line of header.
# Note use of (1,-1) in lines to skip first line and then read to end of file
# Note use of (0,) in columns to pick first column, since its a tuple need trailing comma
x = read_array("test.txt", lines=(1, -1), columns=(0, ))
y = read_array("test.txt", lines=(1, -1), columns=(1, ))
#Second example, read the file into a single arry
z = read_array("test.txt", lines=(1, -1), columns=(0, 2))
# Plot the data
plot(x, y, 'r--', z[:, 0], z[:, 1])
show()
def test_float(self):
a = rand(3, 4) * 30
fname = tempfile.mktemp('.dat')
io.write_array(fname, a)
b = io.read_array(fname)
assert_array_almost_equal(a, b, decimal=4)
os.remove(fname)
def test_float(self):
a = rand(3,4)*30
fname = tempfile.mktemp('.dat')
io.write_array(fname,a)
b = io.read_array(fname)
assert_array_almost_equal(a,b,decimal=4)
os.remove(fname)
def test_complex(self):
a = rand(13,4) + 1j*rand(13,4)
fname = tempfile.mktemp('.dat')
io.write_array(fname,a)
b = io.read_array(fname,atype=N.Complex)
assert_array_almost_equal(a,b,decimal=4)
os.remove(fname)
def make_linelist(files, cutoff, fluxlimit, linefile):
"""
make_linelist(files,cutoff,fluxlimit,linefile)
Cleans/combines linelists.
Inputs:
files - list containing the names of the arc files
cutoff - maximum wavelength
fluxlimit - minimum flux
linefile - name of output file
Outputs:
outputs the data for 'good' arc lines to a single file.
"""
outfile = open(linefile, 'w')
if fluxlimit is None:
fluxlimit = 0.
for name in files:
f = open(name)
lines = sio.read_array(f)
f.close()
for i, j in lines:
if i > cutoff or j < fluxlimit:
continue
else:
outfile.write("%8.3f %8.3f\n" % (i, j))
outfile.close()
def make_arc(filename, dispersion, wave):
"""
make_arc(filename,dispersion,wave)
Create models of the arclamps used.
Inputs:
filename - name of file containing arc wavelengths and amplitudes
dispersion - width of arcs
wave - wavelength scale on which to evaluate arcs
Outputs:
resolution-matched arc spectrum and 3x broadened arc spectrum
"""
f = open(filename)
lines = sio.read_array(f)
f.close()
n = lines.shape[0]
fitmodel = scipy.zeros(3 * n + 1)
fitmodel2 = scipy.zeros(3 * n + 1)
index = 1
for i in range(n):
fitmodel[index] = lines[i, 1]
fitmodel[index + 1] = lines[i, 0]
fitmodel[index + 2] = dispersion
fitmodel2[index] = 1.
fitmodel2[index + 1] = lines[i, 0]
fitmodel2[index + 2] = dispersion * 3.
index += 3
return sf.ngauss(wave, fitmodel), sf.ngauss(wave, fitmodel2)
def test_complex(self):
a = rand(13, 4) + 1j * rand(13, 4)
fname = tempfile.mktemp('.dat')
io.write_array(fname, a)
b = io.read_array(fname, atype=N.Complex)
assert_array_almost_equal(a, b, decimal=4)
os.remove(fname)
def test_integer(self):
from scipy import stats
a = stats.randint.rvs(1, 20, size=(3, 4))
fname = tempfile.mktemp('.dat')
io.write_array(fname, a)
b = io.read_array(fname, atype=a.dtype.char)
assert_array_equal(a, b)
os.remove(fname)
def test_integer(self):
from scipy import stats
a = stats.randint.rvs(1,20,size=(3,4))
fname = tempfile.mktemp('.dat')
io.write_array(fname,a)
b = io.read_array(fname,atype=a.dtype.char)
assert_array_equal(a,b)
os.remove(fname)
def getlines(linefile): from scipy import io as sio file = open(linefile) lines = sio.read_array(file) file.close() lines = lines[:,0] lines.sort() return lines
def aband(inwave,airmass=1.,scale=0.85): path = spectra.__path__[0] file = path+"/data/aband.dat" aband = sio.read_array(file) wave = aband[:,0] data = aband[:,1].astype(scipy.float32) return get_correction(inwave,airmass,scale,wave,data)
def response(wave, file1, file2):
""" Not used. """
from scipy import interpolate
f = open(file1)
resp1 = sio.read_array(f)
f.close()
x = resp1[:, 0]
z = resp1[:, 1]
spline = interpolate.splrep(x, z, s=0)
resp1 = interpolate.splev(wave, spline)
f = open(file2)
resp2 = sio.read_array(f)
f.close()
x = resp2[:, 0]
z = resp2[:, 1]
spline = interpolate.splrep(x, z, s=0)
resp2 = interpolate.splev(wave, spline)
return resp1 / resp2
def main():
""" Build/train/test RBF net
"""
from scipy.io import read_array
print "\nCreating RBF net"
net = rbf(12, 2)
print "\nLoading training and test sets...",
X_trn = read_array('data/oil-trn.dat', columns=(0, (1, 12)), lines=(3, -1))
Y_trn = read_array('data/oil-trn.dat', columns=(12, -1), lines=(3, -1))
X_tst = read_array('data/oil-tst.dat', columns=(0, (1, 12)), lines=(3, -1))
Y_tst = read_array('data/oil-tst.dat', columns=(12, -1), lines=(3, -1))
print "done."
#print "\nInitial SSE:\n"
#print "\ttraining set: ",net.test_all(X_trn,Y_trn)
#print "\ttesting set: ",net.test_all(X_tst,Y_tst),"\n"
print "Training...",
net.train(X_trn, Y_trn)
print "done."
print "\nFinal SSE:\n"
print "\ttraining set: ", net.test_all(X_trn, Y_trn)
print "\ttesting set: ", net.test_all(X_tst, Y_tst), "\n"
def main():
""" Build/train/test RBF net
"""
from scipy.io import read_array
print "\nCreating RBF net"
net = rbf(12,2)
print "\nLoading training and test sets...",
X_trn = read_array('data/oil-trn.dat',columns=(0,(1,12)),lines=(3,-1))
Y_trn = read_array('data/oil-trn.dat',columns=(12,-1),lines=(3,-1))
X_tst = read_array('data/oil-tst.dat',columns=(0,(1,12)),lines=(3,-1))
Y_tst = read_array('data/oil-tst.dat',columns=(12,-1),lines=(3,-1))
print "done."
#print "\nInitial SSE:\n"
#print "\ttraining set: ",net.test_all(X_trn,Y_trn)
#print "\ttesting set: ",net.test_all(X_tst,Y_tst),"\n"
print "Training...",
net.train(X_trn,Y_trn)
print "done."
print "\nFinal SSE:\n"
print "\ttraining set: ",net.test_all(X_trn,Y_trn)
print "\ttesting set: ",net.test_all(X_tst,Y_tst),"\n"
def main():
""" Build/train/test MLP
"""
from scipy.io import read_array, write_array
print "\nCreating 2-2-1 MLP with logistic outputs"
net = mlp(2,2,1,'logistic')
print "\nLoading training and test sets...",
trn_input = read_array('data/xor-trn.dat',lines=(3,-1),columns=(0,(1,2)))
trn_targs = read_array('data/xor-trn.dat',lines=(3,-1),columns=(2,-1))
trn_targs = trn_targs.reshape(N.size(trn_targs),1)
tst_input = read_array('data/xor-tst.dat',lines=(3,-1),columns=(0,(1,2)))
tst_targs = read_array('data/xor-tst.dat',lines=(3,-1),columns=(2,-1))
tst_targs = tst_targs.reshape(N.size(tst_targs),1)
print "done."
print "\nInitial SSE:\n"
print "\ttraining set: ",net.test_all(trn_input,trn_targs)
print "\ttesting set: ",net.test_all(tst_input,tst_targs),"\n"
net.wp = net.train(trn_input,trn_targs)[0]
print "\nFinal SSE:\n"
print "\ttraining set: ",net.test_all(trn_input,trn_targs)
print "\ttesting set: ",net.test_all(tst_input,tst_targs),"\n"
def main():
""" Set up a 1-2-1 SRN to solve the temporal-XOR problem from Elman 1990.
"""
from scipy.io import read_array, write_array
print "\nCreating 1-2-1 SRN for 'temporal-XOR'"
net = srn(1, 2, 1, 'logistic')
print "\nLoading training and test sets...",
trn_input = read_array('data/txor-trn.dat')
trn_targs = N.hstack([trn_input[1:], trn_input[0]])
trn_input = trn_input.reshape(N.size(trn_input), 1)
trn_targs = trn_targs.reshape(N.size(trn_targs), 1)
tst_input = read_array('data/txor-tst.dat')
tst_targs = N.hstack([tst_input[1:], tst_input[0]])
tst_input = tst_input.reshape(N.size(tst_input), 1)
tst_targs = tst_targs.reshape(N.size(tst_targs), 1)
print "done."
print "\nInitial SSE:\n"
print "\ttraining set: ", net.test_all(trn_input, trn_targs)
print "\ttesting set: ", net.test_all(tst_input, tst_targs), "\n"
net.wp = net.train(trn_input, trn_targs)[0]
print "\nFinal SSE:\n"
print "\ttraining set: ", net.test_all(trn_input, trn_targs)
print "\ttesting set: ", net.test_all(tst_input, tst_targs), "\n"
def main():
""" Set up a 1-2-1 SRN to solve the temporal-XOR problem from Elman 1990.
"""
from scipy.io import read_array, write_array
print "\nCreating 1-2-1 SRN for 'temporal-XOR'"
net = srn(1,2,1,'logistic')
print "\nLoading training and test sets...",
trn_input = read_array('data/txor-trn.dat')
trn_targs = N.hstack([trn_input[1:],trn_input[0]])
trn_input = trn_input.reshape(N.size(trn_input),1)
trn_targs = trn_targs.reshape(N.size(trn_targs),1)
tst_input = read_array('data/txor-tst.dat')
tst_targs = N.hstack([tst_input[1:],tst_input[0]])
tst_input = tst_input.reshape(N.size(tst_input),1)
tst_targs = tst_targs.reshape(N.size(tst_targs),1)
print "done."
print "\nInitial SSE:\n"
print "\ttraining set: ",net.test_all(trn_input,trn_targs)
print "\ttesting set: ",net.test_all(tst_input,tst_targs),"\n"
net.wp = net.train(trn_input,trn_targs)[0]
print "\nFinal SSE:\n"
print "\ttraining set: ",net.test_all(trn_input,trn_targs)
print "\ttesting set: ",net.test_all(tst_input,tst_targs),"\n"
def analyze_data(ref_files,transp):
""" Plots histograms of the data.
"""
data = np.zeros((0, 3))
data_ind = []
for n in range(len(ref_files)):
target_file = DATA_PATH + ref_files[n][:-4] + "REF.txt"
tmp = read_array(target_file,',') + transp[n]
data = np.r_[data, tmp]
data_ind.append(tmp)
pitch = data[:,1]
pitch = pitch[ np.where(pitch > 10) ]
unique = list(set(pitch))
unique.sort()
print "pitch classes:",unique
print "nr of classes:",len(unique)+1
xmin = unique[0]
xmax = unique[-1]
xbins = xmax-xmin
# plot global statistics
pylab.figure()
pylab.subplot(211)
pylab.hist(pitch,bins=xbins)
pylab.xlim(xmin,xmax)
pylab.title("global pitch classes histogram")
pylab.subplot(212)
pylab.hist(data[:,2],bins=13)
pylab.xlim(-1,11)
pylab.title("global chromas histogram")
# plot individual pieces
pylab.figure()
nr = len(data_ind)
for n in range(nr):
pitch = data_ind[n][:,1]
pitch = pitch[ np.where(pitch > 10) ]
unique = list(set(pitch))
unique.sort()
print n,":",unique
# xmin = unique[0]
# xmax = unique[-1]
# plot it
pylab.subplot(nr,1,n+1)
pylab.hist(pitch,bins=(xmax-xmin))
pylab.xlim(xmin,xmax)
pylab.title(ref_files[n])
def gist_load(fn):
import string
f = open(fn)
not_done = True
while not_done:
p = f.tell()
s = f.readline()
ss = string.split(s)
if len(ss) == 3 and reduce(lambda b, c: b and c in string.digits,
"".join(ss)):
f.seek(p)
arr = io.read_array(f, atype='b')
not_done = False
return arr
def load(self,fname):
"""
Loading data from a csv or a pickle file of the dbase class
"""
fext = self.__ext(fname)
f = open(fname,'r')
if fext == 'csv':
self.varnm = self.__vardic(f.readline().split(','))
self.data = read_array(f, separator=',', lines=(0,-1))
elif fext == 'pickle':
a = pickle.load(f)
self.varnm = a.varnm
self.data = a.data
else:
raise 'This class only works on csv and pickle files'
f.close()
def load(self, fname):
"""
Loading data from a csv or a pickle file of the dbase class
"""
fext = self.__ext(fname)
f = open(fname, 'r')
if fext == 'csv':
self.varnm = self.__vardic(f.readline().split(','))
self.data = read_array(f, separator=',', lines=(0, -1))
elif fext == 'pickle':
a = pickle.load(f)
self.varnm = a.varnm
self.data = a.data
else:
raise 'This class only works on csv and pickle files'
f.close()
def transform_target_data(audiofile,analyze=0):
""" Gets the target data (in MIDI pitch classes) out of the REF.txt files.
(for my own new datasets)
"""
target_file = DATA_PATH + audiofile[:-4] + "REF.txt"
data = read_array(target_file,',')
if analyze:
pylab.subplot(211)
pylab.plot(data[:,1])
pylab.subplot(212)
pylab.plot(data[:,2])
pylab.figure()
pylab.subplot(211)
pylab.hist(data[:,1],bins=25)
pylab.xlim(40,65)
pylab.subplot(212)
pylab.hist(data[:,2],bins=13)
pylab.xlim(0,12)
# get the unique element (nr. of different notes) of the piece
midi_data = data[:,1]
unique = list(set(midi_data))
target = np.zeros((len(midi_data), len(unique)))
target_chromas = np.zeros((len(midi_data), 13))
for n in range( len(midi_data) ):
ind = unique.index( midi_data[n] )
target[n,ind] = 1
target_chromas[n,data[n,2]+1] = 1
if analyze:
print "pitch classes:",len(unique)
pylab.show()
exit(0)
savefile = OUTPUT_DIR + audiofile[:-4] + "_STFT.dat"
data = shelve.open(savefile)
lenge = data["features"].shape[0]
data["targets"] = target[:lenge]
data["target_midi"] = midi_data[:lenge]
data["target_chromas"] = target_chromas[:lenge]
data["pitch_classes"] = unique
data.close()
def convert_to_midi(reffile,savefile):
""" converts REF files from mirex05 to MIDI data
"""
target_file = DATA_PATH + reffile
data = read_array(target_file)
# remove time axes and convert to midi
midi = np.round( audiotools.freq_to_midi(data[:,1]) )
# calc chromas
chromas = np.ones(len(midi)) * (-1)
for n in range(len(midi)):
if midi[n]>0:
chromas[n] = midi[n] % 12
# write data to disk
writedata = np.c_[data[:,0], midi, chromas]
FILE = open(DATA_PATH + savefile,"w")
write_array(FILE,writedata,',')
FILE.close()
def update_coefficients(self):
""" Uses the samples file to extract parameters using linear multifit """
import pygsl
import pygsl._numobj as Numeric
import pygsl.rng
import pygsl.multifit
from scipy.io import read_array
from numpy import array
data = read_array(file(self['sample_file']))
dep_var = data[:,-3:] # Last 3 columns are supposed to be Renergy, Wenergy, Leakage
indep_var = data[:,:5] # First 3 columns are of interest, 2 are just used as fillers
indep_var[:,-2] = indep_var[:,0] * indep_var[:,1] # We also consider words * bits as a parameter
indep_var[:,-1:] = 1 # Coefficient used as a constant
work = pygsl.multifit.linear_workspace(len(indep_var), 5)
self.c_read, cov, chisq = pygsl.multifit.linear(indep_var, dep_var[:,0], work)
self.c_write, cov, chisq = pygsl.multifit.linear(indep_var, dep_var[:,1], work)
self.c_leakage, cov, chisq = pygsl.multifit.linear(indep_var, dep_var[:,2], work)
def transform_target_data_old(audiofile,analyze=0):
""" Calculates the target data (in MIDI pitch classes) out of the REF.txt files.
(for the old datasets of Ellis and MIREX)
"""
target_file = DATA_PATH + audiofile[:-4] + "REF.txt"
data = read_array(target_file)
# remove time axes and convert to midi
data = data[:,1]
midi_data = audiotools.freq_to_midi(data)
midi_data = np.round(midi_data)
if analyze:
pylab.plot(midi_data)
# get the unique element (nr. of different notes) of the piece
unique = list(set(midi_data))
target = np.zeros((len(midi_data), len(unique)))
for n in range( len(midi_data) ):
ind = unique.index( midi_data[n] )
target[n,ind] = 1
if analyze:
print "classes:",len(unique)
pylab.figure()
pylab.psd(target.flatten())
pylab.show()
exit(0)
savefile = OUTPUT_DIR + audiofile[:-4] + "_STFT.dat"
data = shelve.open(savefile)
lenge = data["features"].shape[0]
data["targets"] = target[:lenge]
data["target_midi"] = midi_data[:lenge]
data["pitch_classes"] = unique
data.close()
#outname=olmodel.RunMaxima('new_sym.tex')
#outname=olmodel.RunMaxima('new_sym_test.tex')
#RunLatex(outname)
runf=1
if runf:
olmodel.CreateFortranandPythonModules('new_sym')
olmodel.CreateFortranandPythonModules('old_sym',newsym=False)
# myfinalfnames=olmodel.PrepareFortranFiles(bodenames)
# mypynames=olmodel.PreparePythonFiles(bodenames)
# fmodnames=olmodel.CallF2py(myfinalfnames)
#olmodel.SymBodeMaximaAll(texname='old_sym.tex',basebodename='old_sym_bode')
dfcomp=read_array(os.path.join(fitdir,'dumbfit_comp_w1.txt'))
dfdb=read_array(os.path.join(fitdir,'dumbfit_db.txt'))
nsp='new_sym'
osp='old_sym'
nl='New Sym'
ol='Old Sym'
myreport.NewBodeDataSetFromFortranModels(nsp, 'new_sym1', nl ,bodenums=[0,1],ucv=dfcomp,expkey='fullsweep')
myreport.NewBodeDataSetFromFortranModels(osp, 'old_sym1', ol ,bodenums=[0,1],ucv=dfcomp,expkey='fullsweep')
myreport.NewBodeDataSetFromFortranModels(nsp, 'new_sym2', nl ,bodenums=[0,1],ucv=dfdb,expkey='fullsweep')
myreport.NewBodeDataSetFromFortranModels(osp, 'old_sym2', ol ,bodenums=[0,1],ucv=dfdb,expkey='fullsweep')
key1='test'
myreport.GenListofBodeFigs(key1,['fullsweep','new_sym1','old_sym1','new_sym2','old_sym2'],[3,3],[212,212],maglims=[[-40,5],[-25,25]],phaselims=[[-400,200],[-220,-50]],magticks=[arange(-40,5,10),arange(-20,25,10)],phaseticks=[arange(-360,220,90),arange(-220,-55,40)])
from numpy import *
from scipy.io import read_array
import qld, quapro
import pylab
from mystic.svc import *
import os.path, time
def myshow():
import Image, tempfile
name = tempfile.mktemp()
pylab.savefig(name,dpi=150)
im = Image.open('%s.png' % name)
im.show()
c1 = read_array(os.path.join('DATA','g1x.pts'))
c2 = read_array(os.path.join('DATA','g2.pts'))
c1[:,0] += 5 # to make the two sets overlap a little
# interchange ?
#c1, c2 = c2, c1
# the Kernel Matrix (with the linear kernel)
# Q = multiply.outer(X,X) <--- unfortunately only works when X is a list of scalars...
# In Mathematica, this would be implemented simply via Outer[K, X, X, 1]
XX = concatenate([c1,-c2])
nx = XX.shape[0]
# quadratic and linear terms of QP
Q = KernelMatrix(XX)
"--max",
action="store_true",
dest="max",
help="show max only?")
options, args = parser.parse_args()
if len(args) != 3:
parser.error("incorrect number of arguments")
print options
jaclistfile = args[0]
outactsfile = args[1]
outfile = args[2]
print "jacobian list file", jaclistfile
print "output activations file", outactsfile
print "output file", outfile
a = io.read_array(file(outactsfile), lines=[2, -1])
labels = file(outactsfile).readline().split()[1:]
T = shape(a)[1]
maxindices = []
for t in range(T):
c = list(a[:, t])
m = c.index(max(c))
if not options.blank and labels[m] == 'blank':
c = c[:-1]
m = c.index(max(c))
maxindices.append(m)
print maxindices
print len(labels)
print labels
if options.bylab:
v = zeros((len(labels), T), 'f')
from scipy.io import read_array
import qld, quapro
import matplotlib.pyplot as plt
from mystic.svc import *
import os.path, time
def myshow():
import Image, tempfile
name = tempfile.mktemp()
plt.savefig(name, dpi=150)
im = Image.open('%s.png' % name)
im.show()
c1 = read_array(os.path.join('DATA', 'g1x.pts'))
c2 = read_array(os.path.join('DATA', 'g2.pts'))
c1[:, 0] += 5 # to make the two sets overlap a little
# interchange ?
#c1, c2 = c2, c1
# the Kernel Matrix (with the linear kernel)
# Q = multiply.outer(X,X) <--- unfortunately only works when X is a list of scalars...
# In Mathematica, this would be implemented simply via Outer[K, X, X, 1]
XX = concatenate([c1, -c2])
nx = XX.shape[0]
# quadratic and linear terms of QP
Q = KernelMatrix(XX)
"""Plot a histogram."""
from pyxgraph import *
from scipy import io
def plot_histogram(epsoutfile, x):
g = pyxgraph(width=6, height=6,
key=None,
xlabel="x",
xlimits=(min(x),max(x))
)
g.pyxplothist(x, Nbins = 100, bin_range=(min(x),max(x)), bars=0)
g.pyxsave(epsoutfile)
if __name__=="__main__":
x = io.read_array("data_histogram.dat")
plot_histogram("histogram.eps", x)
def lris_pipeline(prefix,
dir,
science,
arc,
flats,
out_prefix,
useflat=0,
usearc=0,
cache=0,
offsets=None):
print "Processing mask", out_prefix
scinums = science.split(",")
flatnums = flats.split(",")
for i in range(len(flatnums)):
flatnums[i] = dir + prefix + flatnums[i] + ".fits"
scinames = []
for i in range(len(scinums)):
name = dir + prefix + scinums[i] + ".fits"
scinames.append(name)
arcname = dir + prefix + arc + ".fits"
nsci = len(scinums)
print "Preparing flatfields"
if useflat == 1:
yforw, yback, slits, starboxes, flatnorm = flatload(out_prefix)
else:
yforw, yback, slits, starboxes, flatnorm = flatpipe(
flatnums, out_prefix)
axis1 = flatnorm.shape[0]
axis2 = flatnorm.shape[1]
print "Preparing arcs for line identification"
if usearc == 1:
arcname = out_prefix + "_arc.fits"
arc_tmp = pyfits.open(arcname)
arc_ycor = arc_tmp[0].data.astype(scipy.float32)
lamps = arc_tmp[0].header['LAMPS']
del arc_tmp
else:
arcname = dir + prefix + arc + ".fits"
arc_tmp = pyfits.open(arcname)
arcdata = arc_tmp[0].data.copy()
lamps = arc_tmp[0].header['LAMPS']
del arc_tmp
arcdata = biastrim(arcdata)
arc_ycor = spectools.resampley(arcdata, yforw).astype(scipy.float32)
arcname = out_prefix + "_arc.fits"
arc_hdu = pyfits.PrimaryHDU(arc_ycor)
arc_hdu.header.update('LAMPS', lamps)
arc_hdu.writeto(arcname)
del arc_hdu
wide_stars = []
for i, j in starboxes:
mod = scipy.where((yback < j) & (yback > i))
a = mod[0].min() - 3
b = mod[0].max() + 3
if a < 0:
a = 0
if b > axis1:
b = axis1
wide_stars.append([a, b])
print "Bias trimming and flatfielding science data"
scidata = scipy.zeros((nsci, axis1, axis2), 'f4')
center = scipy.zeros((nsci, len(starboxes)), 'f4')
flux = scipy.zeros((nsci), 'f4')
airmass = []
for i in range(nsci):
filename = scinames[i]
scitmp = pyfits.open(filename)
scidatatmp = scitmp[0].data.copy()
scidatatmp = biastrim(scidatatmp).astype(scipy.float32)
#Remove screwed columns (this should already be done though...)
bad = scipy.where(scidatatmp > 56000.)
nbad = bad[0].size
for k in range(nbad):
y = bad[0][k]
x = bad[1][k]
scidatatmp[y,
x] = (scidatatmp[y, x - 1] + scidatatmp[y, x + 1]) / 2.
# Don't flatfield blueside data
scidatatmp = scidatatmp / flatnorm
scidata[i, :, :] = scidatatmp.copy()
try:
mswave = scitmp[0].header['MSWAVE']
except:
mswave = 6500.
if len(slits) == 1:
try:
mswave = scitmp[0].header['WAVELEN']
except:
pass
disperser = scitmp[0].header['GRANAME']
airmass.append(scitmp[0].header['AIRMASS'])
# Old data mightn't have a dichroic keyword!
try:
dichroic = scitmp[0].header['DICHNAME']
except:
dichroic = None
flux[i] = scipy.sort(scipy.ravel(scidatatmp))[scidatatmp.size / 4]
for j in range(len(starboxes)):
a, b = starboxes[j]
m, n = wide_stars[j]
a -= 4
b += 4
m -= 2
n += 2
center[i, j] = offset.findoffset(scidatatmp[m:n], yforw[a:b], m)
del scitmp
del scidatatmp
del flatnorm
if offsets is not None:
center = scipy.asarray(offsets)
else:
center = stats.stats.nanmean(center, axis=1)
center[scipy.isnan(center)] = 0.
print "Normalizing Fluxes"
cmax = center.max()
fmax = flux.max()
for i in range(center.size):
center[i] -= cmax
ratio = fmax / flux[i]
scidata[i] *= ratio
cmax = ceil(fabs(center.min()))
if disperser == "150/7500":
scale = 4.8
elif disperser == "300/5000":
scale = 2.45
elif disperser == "400/8500":
scale = 1.85
elif disperser == "600/5000":
scale = 1.25
elif disperser == "600/7500":
scale = 1.25
elif disperser == "600/10000":
scale = 1.25
elif disperser == "831/8200":
scale = 0.915
elif disperser == "900/5500":
scale = 0.85
elif disperser == "1200/7500":
scale = 0.64
if dichroic == 'mirror':
redcutoff = 4000.
dich_file = ''
elif dichroic == '460':
redcutoff = 4600.
dich_file = '460'
elif dichroic == '500':
redcutoff = 5000.
dich_file = '500'
elif dichroic == '560':
redcutoff = 5500.
dich_file = '560'
elif dichroic == '680':
redcutoff = 6700.
dich_file = '680'
else:
redcutoff = 3500.
dich_file = ''
nsize = 0
csize = 0
wide_slits = []
linewidth = []
slits = [[1150, 1250]]
for i, j in slits:
csize += int(j - i + cmax) + 5
nsize += j - i + 5
mod = scipy.where((yback > i) & (yback < j))
a = mod[0].min() - 4
b = mod[0].max() + 4
if a < 0:
a = 0
if b > axis1:
b = axis1
wide_slits.append([a, b])
if len(wide_slits) % 7 == 0 or len(slits) == 1:
linewidth.append(
measure_width.measure(arc_ycor[(i + j) / 2, :], 15))
csize -= 5
nsize -= 5
linewidth = scipy.median(scipy.asarray(linewidth))
print "Loading wavelength model"
lris_path = lris_red.__path__[0]
filename = lris_path + "/uves_sky.model"
infile = open(filename, "r")
wavecalmodel = load(infile)
infile.close()
wave = scipy.arange(3400., 10400., 0.1)
if dich_file != '':
filename = lris_path + "/dichroics/dichroic_" + dich_file + "_t.dat"
infile = open(filename, "r")
input = sio.read_array(infile)
infile.close()
spline = interpolate.splrep(input[:, 0], input[:, 1])
dich = interpolate.splev(wave, spline)
dich[wave < 4500.] = 1.
dich[wave > 8800.] = 1.
del input, spline
else:
dich = scipy.ones(wave.size)
wavemodel = interpolate.splev(wave, wavecalmodel)
finemodel = ndimage.gaussian_filter1d(wavemodel, linewidth * scale / 0.1)
wavemodel = ndimage.gaussian_filter1d(finemodel, 5. / 0.1)
finemodel *= dich
finemodel = interpolate.splrep(wave, finemodel)
wavemodel *= dich
widemodel = interpolate.splrep(wave, wavemodel)
goodmodel = finemodel
del dich, wave, wavemodel
extractwidth = 10
print "Creating output arrays"
outlength = int(axis2 * 1.6)
out = scipy.zeros((nsci, nsize, outlength), scipy.float32) * scipy.nan
out2 = scipy.zeros((2, csize, outlength), scipy.float32) * scipy.nan
if cache:
print "Caching..."
strtfile = out_prefix + "_TMPSTRT.fits"
bgfile = out_prefix + "_TMPBSUB.fits"
try:
os.remove(strtfile)
except:
pass
outfile = pyfits.PrimaryHDU(out)
outfile.header.update('CTYPE1', 'LINEAR')
outfile.header.update('CRPIX1', 1)
outfile.header.update('CRVAL1', mswave - (0.5 * out2.shape[2]) * scale)
outfile.header.update('CD1_1', scale)
outfile.header.update('CTYPE2', 'LINEAR')
outfile.header.update('CRPIX2', 1)
outfile.header.update('CRVAL2', 1)
outfile.header.update('CD2_2', 1)
if nsci > 1:
outfile.header.update('CTYPE3', 'LINEAR')
outfile.header.update('CRPIX3', 1)
outfile.header.update('CRVAL3', 1)
outfile.header.update('CD3_3', 1)
outfile.writeto(strtfile)
del outfile, out
try:
os.remove(bgfile)
except:
pass
outfile = pyfits.PrimaryHDU(out2)
outfile.header.update('CTYPE1', 'LINEAR')
outfile.header.update('CRPIX1', 1)
outfile.header.update('CRVAL1', mswave - (0.5 * out2.shape[2]) * scale)
outfile.header.update('CD1_1', scale)
outfile.header.update('CTYPE2', 'LINEAR')
outfile.header.update('CRPIX2', 1)
outfile.header.update('CRVAL2', 1)
outfile.header.update('CD2_2', 1)
outfile.header.update('CTYPE3', 'LINEAR')
outfile.header.update('CRPIX3', 1)
outfile.header.update('CRVAL3', 1)
outfile.header.update('CD3_3', 1)
outfile.writeto(bgfile)
del outfile, out2
posc = 0
posn = 0
count = 1
for k in range(len(slits)):
i, j = slits[k]
a, b = wide_slits[k]
##
if count < 1:
count += 1
continue
##
print "Working on slit %d (%d to %d)" % (count, i, j)
sky2x, sky2y, ccd2wave = wavematch(a, scidata[:, a:b], arc_ycor[i:j],
yforw[i:j], widemodel, finemodel,
goodmodel, scale, mswave, redcutoff)
strt, bgsub, varimg = doskysub(i, j - i, outlength, scidata[:, a:b],
yback[a:b], sky2x, sky2y, ccd2wave,
scale, mswave, center, redcutoff,
airmass)
h = strt.shape[1]
if cache:
file = pyfits.open(strtfile, mode="update")
out = file[0].data
out[:, posn:posn + h] = strt.copy()
if cache:
file.close()
del file, out
posn += h + 5
##
# lris_red.skysub.RESAMPLE = 1
# count += 1
# continue
##
h = bgsub.shape[0]
if cache:
file = pyfits.open(bgfile, mode="update")
out2 = file[0].data
out2[0, posc:posc + h] = bgsub.copy()
out2[1, posc:posc + h] = varimg.copy()
if cache:
file.close()
del file, out2
posc += h + 5
##
# count += 1
# continue
##
tmp = scipy.where(scipy.isnan(bgsub), 0., bgsub)
filter = tmp.sum(axis=0)
mod = scipy.where(filter != 0)
start = mod[0][0]
end = mod[0][-1] + 1
del tmp
slit = bgsub[:, start:end]
spectra = extract(slit, varimg[:, start:end], extractwidth)
num = 1
crval = mswave - (0.5 * bgsub.shape[1] - start) * scale
for spec in spectra:
for item in spec:
if item.size == 4:
hdu = pyfits.PrimaryHDU()
hdu.header.update('CENTER', item[2])
hdu.header.update('WIDTH', item[3])
hdulist = pyfits.HDUList([hdu])
else:
thdu = pyfits.ImageHDU(item)
thdu.header.update('CRVAL1', crval)
thdu.header.update('CD1_1', scale)
thdu.header.update('CRPIX1', 1)
thdu.header.update('CRVAL2', 1)
thdu.header.update('CD2_2', 1)
thdu.header.update('CRPIX2', 1)
thdu.header.update('CTYPE1', 'LINEAR')
hdulist.append(thdu)
outname = out_prefix + "_spec_%02d_%02d.fits" % (count, num)
hdulist.writeto(outname)
num += 1
count += 1
##
# file = pyfits.open(bgfile)
# file.writeto(out_prefix+"_save.fits")
# return
##
if cache:
file = pyfits.open(bgfile)
out2 = file[0].data.copy()
del file
tmp = out2[0].copy()
tmp = scipy.where(scipy.isnan(tmp), 0, 1)
mod = scipy.where(tmp.sum(axis=0) != 0)
start = mod[0][0]
end = mod[0][-1] + 1
del tmp
outname = out_prefix + "_bgsub.fits"
outfile = pyfits.PrimaryHDU(out2[0, :, start:end])
outfile.header.update('CTYPE1', 'LINEAR')
outfile.header.update('CRPIX1', 1)
outfile.header.update('CRVAL1',
mswave - (0.5 * out2.shape[2] - start) * scale)
outfile.header.update('CD1_1', scale)
outfile.header.update('CRPIX2', 1)
outfile.header.update('CRVAL2', 1)
outfile.header.update('CD2_2', 1)
outfile.writeto(outname)
hdr = outfile.header.copy()
outname = out_prefix + "_var.fits"
outfile = pyfits.PrimaryHDU(out2[1, :, start:end])
outfile.header = hdr
outfile.writeto(outname)
del out2, hdr
if cache:
file = pyfits.open(strtfile)
out = file[0].data.copy()
del file
outname = out_prefix + "_straight.fits"
outfile = pyfits.PrimaryHDU(out[:, :, start:end])
outfile.header.update('CTYPE1', 'LINEAR')
outfile.header.update('CRPIX1', 1)
outfile.header.update('CRVAL1',
mswave - (0.5 * out.shape[2] - start) * scale)
outfile.header.update('CD1_1', scale)
outfile.header.update('CRPIX2', 1)
outfile.header.update('CRVAL2', 1)
outfile.header.update('CD2_2', 1)
if nsci > 1:
outfile.header.update('CRPIX3', 1)
outfile.header.update('CRVAL3', 1)
outfile.header.update('CD3_3', 1)
outfile.writeto(outname)
del out, outfile
from __future__ import division
from past.utils import old_div
from scipy import *
import scipy.io as io
from SloppyCell.ReactionNetworks import *
alb_dat = io.read_array('Brodersen_1987/Albumin.ens.dat')
alb_dat = transpose(reshape(alb_dat, (6, 30)))
ua, va = Ensembles.PCA_eig_log_params(alb_dat)
alb_chi2_hess, temp, temp = Utility.load('Brodersen_1987/hess_dict.albumin.bp')
ua2, va2 = Utility.eig(alb_chi2_hess)
heme_dat = io.read_array('Brodersen_1987/Hemoglobin.ens.dat')
heme_dat = transpose(reshape(heme_dat, (4, 30)))
uh, vh = Ensembles.PCA_eig_log_params(heme_dat)
heme_chi2_hess, temp, temp = Utility.load(
'Brodersen_1987/hess_dict.hemeglobin.bp')
uh2, vh2 = Utility.eig(heme_chi2_hess)
Plotting.figure(figsize=(6, 3))
Plotting.plot_eigval_spectrum(old_div(ua, ua[0]), widths=0.8)
Plotting.plot_eigval_spectrum(old_div(ua2, ua2[0]), widths=0.8, offset=1.0)
Plotting.plot_eigval_spectrum(old_div(uh, uh[0]), widths=0.8, offset=2.0)
Plotting.plot_eigval_spectrum(old_div(uh2, uh2[0]), widths=0.8, offset=3.0)
Plotting.gca().set_xlim(-0.1, 3.9)
Plotting.gca().set_ylim(0.5e-6, 2)
Plotting.ylabel(r'$\lambda/\lambda_0$', fontsize='large')
Plotting.xticks([0.4, 1.4, 2.4, 3.4],
#!/usr/bin/python
# Author: Varun Hiremath <*****@*****.**>
# Created: Thu, 2 Apr 2009 05:21:30 -0400
import scipy, pylab
from scipy import io
import plot_settings as ps
error = io.read_array("error.op")
nrs = error[:,0]
# Publishable quality image
ps.set_mode("publish")
pylab.figure(1)
pylab.axes([0.125,0.2,0.95-0.125,0.95-0.3])
for i in range(6, len(error[0])):
pylab.plot(nrs, error[:,i], ps.lps[i], label="$\Phi_{"+str(i-5)+"}$")
pylab.legend(numpoints=1)
pylab.xlabel("$n_{rs}$")
pylab.ylabel("error ($\epsilon$)")
pylab.title("Plot of error")
pylab.savefig("error_publish.eps")
pylab.savefig("error_publish.png")
# Medium size image
ps.set_mode("medium")
pylab.figure(2)
# pylab.axes([0.125,0.2,0.95-0.125,0.95-0.3])
def lris_pipeline(prefix,
dir,
scinames,
arcname,
flatnames,
out_prefix,
useflat=0,
usearc=0,
cache=0,
offsets=None,
logfile=None):
# Create a logfile for this session
if logfile is None:
logfile = open('%s.log' % out_prefix, 'w')
else:
logfile = open(logfile, 'w')
stime = time.strftime("%d/%m/%y %H:%M:%S")
logfile.write('%s\n' % stime)
print "Processing mask", out_prefix
logfile.write('Processing mask %s\n' % out_prefix)
""" Prepare image names. """
nsci = len(scinames)
YMID = 2048 # offset for the second detector
print "Preparing flatfields"
if useflat == 1:
logfile.write('Using pre-made flats\n')
yforw, yback, slits, starboxes = flatload(out_prefix)
else:
logfile.write('Making new flats\n')
yforw, yback, slits, starboxes = flatpipe(flatnames, out_prefix)
print "Preparing arcs for line identification"
if usearc == 1:
logfile.write('Using pre-made arcs\n')
arc_ycor = {}
for i in ['bottom', 'top']:
arcname = out_prefix + "_arc_%s.fits" % i
arc_tmp = pyfits.open(arcname)
arc_ycor[i] = arc_tmp[0].data.astype(scipy.float32)
lamps = arc_tmp[0].header['LAMPS']
filter = arc_tmp[0].header['BLUFILT']
del arc_tmp
else:
logfile.write('Making new arcs\n')
""" Load arc data from fits file """
arc_tmp = pyfits.open(arcname)
arcdata = arc_tmp[0].data.copy()
""" Determine which lamps were used """
lamps = arc_tmp[0].header['LAMPS']
try:
filter = arc_tmp[0].header['BLUFILT']
except:
filter = 'clear'
del arc_tmp
""" Process arcs for the top and bottom separately """
arcdata = biastrim(arcdata)
arc_ycor = {}
arc_ycor['bottom'] = spectools.resampley(
arcdata[:YMID], yforw['bottom']).astype(scipy.float32)
arcname = out_prefix + "_arc_bottom.fits"
arc_hdu = pyfits.PrimaryHDU(arc_ycor['bottom'])
arc_hdu.header.update('LAMPS', lamps)
arc_hdu.header.update('BLUFILT', filter)
arc_hdu.writeto(arcname)
arc_ycor['top'] = spectools.resampley(
arcdata[YMID:], yforw['top']).astype(scipy.float32)
arcname = out_prefix + "_arc_top.fits"
arc_hdu = pyfits.PrimaryHDU(arc_ycor['top'])
arc_hdu.header.update('LAMPS', lamps)
arc_hdu.header.update('BLUFILT', filter)
arc_hdu.writeto(arcname)
del arc_hdu, arcdata
axis1 = 4096
axis2 = 4096
"""
We create 'wide' starboxes that describe the minimum and maximum
y-position of the slit in the *unstraightened* frame.
"""
wide_stars = {}
logfile.write('\n')
for l in ['bottom', 'top']:
wide_stars[l] = []
bmax = arc_ycor[l].shape[0]
logfile.write('Star boxes for %s\n' % l)
for i, j in starboxes[l]:
logfile.write('[:,%d:%d]\n' % (i, j))
mod = scipy.where((yback[l] < j) & (yback[l] > i))
a = mod[0].min() - 3 # We include a small amount of
b = mod[0].max() + 3 # padding for resampling.
if a < 0:
a = 0
if b > bmax:
b = bmax
wide_stars[l].append([a, b])
print "Bias trimming science data"
nstars = len(starboxes['bottom']) + len(starboxes['top'])
scidata = scipy.zeros((nsci, axis1, axis2), 'f4')
center = scipy.zeros((nsci, nstars), 'f4')
flux = scipy.zeros((nsci), 'f4')
for i in range(nsci):
filename = scinames[i]
scitmp = pyfits.open(filename)
scidatatmp = scitmp[0].data.copy()
scidatatmp = biastrim(scidatatmp).astype(scipy.float32)
"""
Remove screwed columns (this should already be done by biastrim
though...).
"""
bad = scipy.where(scidatatmp > 56000.)
nbad = bad[0].size
for k in range(nbad):
y = bad[0][k]
x = bad[1][k]
scidatatmp[y,
x] = (scidatatmp[y, x - 1] + scidatatmp[y, x + 1]) / 2.
"""
We don't flatfield blueside data because of ghosts and
reflections. Milan Bogosavljevic has data that show that
flatfielding is important if using very blue data--the bluest
end looks like it has fringes! I should add a flag to allow
flatfielding to be turned on....
"""
scidata[i, :, :] = scidatatmp.copy()
"""
The try/except code is because sometimes the data just don't
have these keywords (I think this might be correlated to
stopping exposures early, though I'm not sure). Plus old data
might not have used the dichroic at all....
"""
try:
disperser = scitmp[0].header['GRISNAME']
except:
pass
try:
dichroic = scitmp[0].header['DICHNAME']
except:
dichroic = None
"""
We use the first quartile for the flux normalization.
"""
flux[i] = scipy.sort(scipy.ravel(scidatatmp))[scidatatmp.size / 4]
"""
The starboxes are used to determine relative y-shifts between
mask exposures. If the offsets keyword was used, this will
be ignored.
"""
for l in ['bottom', 'top']:
if offsets is not None:
continue
for j in range(len(starboxes[l])):
a, b = starboxes[l][j]
m, n = wide_stars[l][j]
a -= 4
b += 4
m -= 2
n += 2
if a < 0:
a = 0
if l == 'top':
m += YMID
n += YMID
center[i, j] = offset.findoffset(scidatatmp[m:n],
yforw[l][a:b], m)
del scitmp
del scidatatmp
if offsets is not None:
center = scipy.asarray(offsets)
else:
center = stats.stats.nanmean(center, axis=1)
center[scipy.isnan(center)] = 0.
print "Normalizing Fluxes"
cmax = center.max()
fmax = flux.max()
logfile.write('\nMask pixel and flux offsets\n')
logfile.write('-----------------------------\n')
logfile.write('Mask Pixels Flux\n')
for i in range(center.size):
center[i] -= cmax
ratio = fmax / flux[i]
scidata[i] *= ratio
logfile.write('%4d %6.2f %4.2f\n' % (i, center[i], ratio))
cmax = ceil(fabs(center.min()))
logfile.write('\n')
if disperser == "300/5000":
scale = 1.41
mswave = 5135.
elif disperser == "400/3400":
scale = 1.05
mswave = 3990.
elif disperser == "600/4000":
scale = 0.63
mswave = 4590.
elif disperser == "1200/3400":
scale = 0.24
mswave = 3505.
if dichroic == 'mirror':
bluecutoff = 0.
dich_file = ''
elif dichroic == '460':
bluecutoff = 4650.
dich_file = '460'
elif dichroic == '500':
bluecutoff = 5100.
dich_file = '500'
elif dichroic == '560':
bluecutoff = 5650.
dich_file = '560'
elif dichroic == '680':
# bluecutoff = 6800.
bluecutoff = 5650.
dich_file = '680'
else:
bluecutoff = 8000.
dich_file = ''
"""
We create 'wide' slits that describe the minimum and maximum y-position
of the slit in the *unstraightened* frame. We also determine the mask
resolution using every seventh slit.
"""
nsize = 0 # Size of straightened mask
csize = 0 # Size of coadded mask
wide_slits = {}
linewidth = []
print "Determining mask resolution"
for l in ['bottom', 'top']:
wide_slits[l] = []
bmax = arc_ycor[l].shape[0]
logfile.write('Slits for %s (%d total)\n' % (l, len(slits[l])))
for i, j in slits[l]:
logfile.write('[:,%d:%d]\n' % (i, j))
csize += int(j - i + cmax) + 5
nsize += j - i + 5
mod = scipy.where((yback[l] > i) & (yback[l] < j))
a = mod[0].min() - 4
b = mod[0].max() + 4
if a < 0:
a = 0
if b > bmax:
b = bmax
wide_slits[l].append([a, b])
if len(wide_slits[l]) % 7 == 0:
linewidth.append(
measure_width.measure(arc_ycor[l][(i + j) / 2, :], 15))
csize -= 5
nsize -= 5
logfile.write("\n\n")
logfile.close()
linewidth = scipy.median(scipy.asarray(linewidth))
""" We can temporarily delete the top CCD from memory """
yforw = yforw['bottom']
yback = yback['bottom']
arc_ycor = arc_ycor['bottom']
"""
Create the arclamp model by using only the blue lamps that were turned
on. Turning off the red lamps (Ne and Ar) reduces reflections on the
blue side, and it is difficult to use these lines for the wavelength
solution because 2nd order blue lines show up starting at ~5800.
"""
print "Loading wavelength model"
lris_path = lris.__path__[0]
lamps = lamps.split(',')
wave = scipy.arange(2000., 8000., 0.1)
filenames = []
if lamps[0] == '1':
filenames.append(lris_path + "/data/bluearcs/hg.dat")
if lamps[3] == '1':
filenames.append(lris_path + "/data/bluearcs/cd.dat")
if lamps[4] == '1':
filenames.append(lris_path + "/data/bluearcs/zn.dat")
fluxlimit = None
if filter == 'SP580' and bluecutoff > 5650:
cutoff = 5650.
else:
fluxlimit = 150.
cutoff = bluecutoff
linefile = out_prefix + "_lines.dat"
make_linelist(filenames, cutoff, fluxlimit, linefile)
"""
The relative amplitudes of the lines in the hg, cd, and zn.dat files
are more appropriate for the bluer grisms. A separate linelist is
used for the 300 grism and assumes all three lamps were on. This is
one of the problems with (1) not knowing the throughput for each
setup and (2) not having stable lamps.
"""
if disperser == "300/5000":
filename = lris_path + "/data/bluearcs/300_lines.dat"
arc, lines = make_arc(filename, linewidth * scale, wave)
else:
arc, lines = make_arc(linefile, linewidth * scale, wave)
finemodel = interpolate.splrep(wave, arc, s=0)
smooth = ndimage.gaussian_filter1d(arc, 9. / 0.1)
widemodel = interpolate.splrep(wave, smooth, s=0)
linemodel = interpolate.splrep(wave, lines, s=0)
filename = lris_path + "/data/uves_sky.model"
infile = open(filename, "r")
wavecalmodel = pickle.load(infile)
infile.close()
wave = scipy.arange(3400., 10400., 0.1)
""" We attempt to model the dichroic cutoff for the sky mode. """
if dichroic == '680' and disperser == "300/5000":
filename = lris_path + "/data/dichroics/dichroic_680_t.dat"
infile = open(filename, "r")
input = sio.read_array(infile)
infile.close()
input[:, 1] = 1. - input[:, 1]
spline = interpolate.splrep(input[:, 0], input[:, 1], s=0)
dich = interpolate.splev(wave, spline)
dich[wave < 4500.] = 1.
dich[wave > 8800.] = 1.
filename = lris_path + "/data/grisms/grism_300.dat"
infile = open(filename, "r")
input = sio.read_array(infile)
infile.close()
spline = interpolate.splrep(input[:, 0], input[:, 1], s=0)
eff = interpolate.splev(wave, spline)
eff[wave < 5100.] = 1.
eff[wave > 7200.] = 1.
dich *= eff
del input, spline
else:
dich = scipy.ones(wave.size)
wave = scipy.arange(3400., 10400., 0.1)
wavemodel = interpolate.splev(wave, wavecalmodel)
goodmodel = ndimage.gaussian_filter1d(wavemodel, linewidth * scale / 0.12)
goodmodel *= dich
goodmodel = interpolate.splrep(wave, goodmodel, s=0)
extra = [linefile, cutoff, 3000]
extractwidth = 15
del arc, wave, smooth
"""
Use the skyarcmatch routine if the 300grism is employed, otherwise
just match the arclines.
"""
if dichroic == '680' and disperser == "300/5000":
from lris.lris_blue.skyarcmatch import arcmatch as wavematch
extra2 = [linefile, 6850, 3500]
else:
from lris.lris_blue.arcmatch import arcmatch as wavematch
extra2 = extra
"""
This could be improved by making the arrays the (pre-determined) size
stipulated by the red and blue cutoffs.
"""
print "Creating output arrays"
outlength = int(axis2 * 1.6)
out = scipy.zeros((nsci, nsize, outlength), scipy.float32) * scipy.nan
out2 = scipy.zeros((2, csize, outlength), scipy.float32) * scipy.nan
"""
For systems with limited RAM, it might make sense to cache the output
arrays to disk. This increases the time it takes to run but may be
necessary and also allows the progress of the reduction to be
monitored.
"""
if cache:
import os
print "Caching..."
strtfile = out_prefix + "_TMPSTRT.fits"
bgfile = out_prefix + "_TMPBSUB.fits"
try:
os.remove(strtfile)
except:
pass
outfile = pyfits.PrimaryHDU(out)
outfile.header.update('CTYPE1', 'LINEAR')
outfile.header.update('CRPIX1', 1)
outfile.header.update('CRVAL1', mswave - (0.5 * out2.shape[2]) * scale)
outfile.header.update('CD1_1', scale)
outfile.header.update('CTYPE2', 'LINEAR')
outfile.header.update('CRPIX2', 1)
outfile.header.update('CRVAL2', 1)
outfile.header.update('CD2_2', 1)
if nsci > 1:
outfile.header.update('CTYPE3', 'LINEAR')
outfile.header.update('CRPIX3', 1)
outfile.header.update('CRVAL3', 1)
outfile.header.update('CD3_3', 1)
outfile.writeto(strtfile)
del outfile, out
try:
os.remove(bgfile)
except:
pass
outfile = pyfits.PrimaryHDU(out2)
outfile.header.update('CTYPE1', 'LINEAR')
outfile.header.update('CRPIX1', 1)
outfile.header.update('CRVAL1', mswave - (0.5 * out2.shape[2]) * scale)
outfile.header.update('CD1_1', scale)
outfile.header.update('CTYPE2', 'LINEAR')
outfile.header.update('CRPIX2', 1)
outfile.header.update('CRVAL2', 1)
outfile.header.update('CD2_2', 1)
outfile.header.update('CTYPE3', 'LINEAR')
outfile.header.update('CRPIX3', 1)
outfile.header.update('CRVAL3', 1)
outfile.header.update('CD3_3', 1)
outfile.writeto(bgfile)
del outfile, out2
logfile = open(logfile.name, 'a')
logfile.write('Beginning Wavelength Solution and Resampling\n')
logfile.write('--------------------------------------------\n')
logfile.close()
"""
Loop through all of the slits, determining the wavelength solution and
performing the background subtraction. It might be more robust to
determine all wavelength solutions, then jointly determine a 'master'
solution.... posc stores the current (starting) position of the
coadded array, and posn stores the current position of the straight
array. off is 0 while looping over the bottom and YMID for the top. n
is the number of bottom slits, so that the 1st top slit is n+1.
"""
nbottom = len(slits['bottom'])
nslits = nbottom + len(slits['top'])
posc = 0
posn = 0
count = 1
off = 0
n = 0
narrow = slits['bottom']
wide = wide_slits['bottom']
""" Debugging feature; set to 1 to skip background subtraction """
lris.lris_blue.skysub.RESAMPLE = 0
for k in range(nslits):
"""
When we have finished all of the bottom slits, switch
parameters over to their top values.
"""
if k == nbottom:
arc_ycor = get_arc(out_prefix)
yforw = get_yforw(out_prefix)
yback = get_yback(out_prefix)
n = nbottom
off = YMID
narrow = slits['top']
wide = wide_slits['top']
i, j = narrow[k - n]
a, b = wide[k - n]
""" Debugging feature; change number to skip initial slits """
if count < 1:
count += 1
continue
print "Working on slit %d (%d to %d)" % (count, i + off, j + off)
logfile = open(logfile.name, 'a')
logfile.write("Working on slit %d (%d to %d)\n" %
(count, i + off, j + off))
logfile.close()
sky2x, sky2y, ccd2wave = wavematch(a, scidata[:, a + off:b + off],
arc_ycor[i:j], yforw[i:j],
widemodel, finemodel, goodmodel,
linemodel, scale, mswave, extra,
logfile)
logfile = open(logfile.name, 'a')
logfile.write("\n")
logfile.close()
strt, bgsub, varimg = doskysub(i, j - i, outlength,
scidata[:, a + off:b + off], yback[a:b],
sky2x, sky2y, ccd2wave, scale, mswave,
center, extra2)
""" Store the resampled 2d spectra """
h = strt.shape[1]
if cache:
file = pyfits.open(strtfile, mode="update")
out = file[0].data
out[:, posn:posn + h] = strt.copy()
if cache:
file.close()
del file, out
posn += h + 5
if lris.lris_blue.skysub.RESAMPLE == 1:
count += 1
continue
""" Store the resampled, background subtracted 2d spectra """
h = bgsub.shape[0]
if cache:
file = pyfits.open(bgfile, mode="update")
out2 = file[0].data
out2[0, posc:posc + h] = bgsub.copy()
out2[1, posc:posc + h] = varimg.copy()
if cache:
file.close()
del file, out2
posc += h + 5
""" Find and extract object traces """
tmp = scipy.where(scipy.isnan(bgsub), 0., bgsub)
filter = tmp.sum(axis=0)
mod = scipy.where(filter != 0)
start = mod[0][0]
end = mod[0][-1] + 1
del tmp
slit = bgsub[:, start:end]
spectra = extract(slit, varimg[:, start:end], extractwidth)
num = 1
crval = mswave - (0.5 * bgsub.shape[1] - start) * scale
for spec in spectra:
for item in spec:
if item.size == 4:
hdu = pyfits.PrimaryHDU()
hdu.header.update('CENTER', item[2])
hdu.header.update('WIDTH', item[3])
hdulist = pyfits.HDUList([hdu])
else:
thdu = pyfits.ImageHDU(item)
thdu.header.update('CRVAL1', crval)
thdu.header.update('CD1_1', scale)
thdu.header.update('CRPIX1', 1)
thdu.header.update('CRVAL2', 1)
thdu.header.update('CD2_2', 1)
thdu.header.update('CRPIX2', 1)
thdu.header.update('CTYPE1', 'LINEAR')
hdulist.append(thdu)
outname = out_prefix + "_spec_%02d_%02d.fits" % (count, num)
hdulist.writeto(outname)
num += 1
count += 1
""" Output 2d spectra"""
if cache:
file = pyfits.open(bgfile)
out2 = file[0].data.copy()
del file
tmp = out2[0].copy()
tmp = scipy.where(scipy.isnan(tmp), 0, 1)
mod = scipy.where(tmp.sum(axis=0) != 0)
start = mod[0][0]
end = mod[0][-1] + 1
del tmp
outname = out_prefix + "_bgsub.fits"
outfile = pyfits.PrimaryHDU(out2[0, :, start:end])
outfile.header.update('CTYPE1', 'LINEAR')
outfile.header.update('CRPIX1', 1)
outfile.header.update('CRVAL1',
mswave - (0.5 * out2.shape[2] - start) * scale)
outfile.header.update('CD1_1', scale)
outfile.header.update('CRPIX2', 1)
outfile.header.update('CRVAL2', 1)
outfile.header.update('CD2_2', 1)
outfile.writeto(outname)
hdr = outfile.header.copy()
outname = out_prefix + "_var.fits"
outfile = pyfits.PrimaryHDU(out2[1, :, start:end])
outfile.header = hdr
outfile.writeto(outname)
del out2, hdr
if cache:
file = pyfits.open(strtfile)
out = file[0].data.copy()
del file
outname = out_prefix + "_straight.fits"
outfile = pyfits.PrimaryHDU(out[:, :, start:end])
outfile.header.update('CTYPE1', 'LINEAR')
outfile.header.update('CRPIX1', 1)
outfile.header.update('CRVAL1',
mswave - (0.5 * out.shape[2] - start) * scale)
outfile.header.update('CD1_1', scale)
outfile.header.update('CRPIX2', 1)
outfile.header.update('CRVAL2', 1)
outfile.header.update('CD2_2', 1)
if nsci > 1:
outfile.header.update('CRPIX3', 1)
outfile.header.update('CRVAL3', 1)
outfile.header.update('CD3_3', 1)
outfile.writeto(outname)
del out, outfile
parser.add_option("-l", "--bylab", action="store_true", dest="bylab", help="calculate score by label?")
parser.add_option("-b", "--blank", action="store_true", dest="blank", help="allow blank?")
parser.add_option("-x", "--softmax", action="store_true", dest="softmax", help="apply softmax?")
parser.add_option("-m", "--max", action="store_true", dest="max", help="show max only?")
options, args = parser.parse_args()
if len(args) != 3:
parser.error("incorrect number of arguments")
print options
jaclistfile = args[0]
outactsfile = args[1]
outfile = args[2]
print "jacobian list file", jaclistfile
print "output activations file", outactsfile
print "output file", outfile
a = io.read_array(file(outactsfile), lines=[2,-1])
labels = file(outactsfile).readline().split()[1:]
T = shape(a)[1]
maxindices = []
for t in range(T):
c = list(a[:,t])
m = c.index(max(c))
if not options.blank and labels[m] == 'blank':
c = c[:-1]
m = c.index(max(c))
maxindices.append(m)
print maxindices
print len(labels)
print labels
if options.bylab:
v = zeros((len(labels),T),'f')
def load_data():
""" Reads the Japanese Vowel Dataset from textfiles as in Jaegers paper
and puts it into python lists / numpy arrays.
"""
aetrain = io.read_array("./JaegerPaper/data/ae.train")
aetest = io.read_array("./JaegerPaper/data/ae.test")
# value for bias input
bias = 0.07
# aetrain and aetest contain the 12-dim time series, which have
# different lengthes, concatenated vertically and separated by ones(1,12)
# rows. We now sort them into lists, such that each element represents
# one time series
trainInputs = []
readindex = 0
for c in range(270):
l = 0
while aetrain[readindex, 0] != 1.0:
l += 1
readindex += 1
readindex += 1
trainInputs.append(aetrain[readindex - l - 1:readindex - 1, :])
# add bias input and input which indicates the length
trainInputs[c] = np.c_[trainInputs[c], bias * np.ones((l, 1)),
(l / 30.) * np.ones((l, 1))]
# now the same with the test inputs
testInputs = []
readindex = 0
for c in range(370):
l = 0
while aetest[readindex, 0] != 1.0:
l += 1
readindex += 1
readindex += 1
testInputs.append(aetest[readindex - l - 1:readindex - 1, :])
# add bias input and input which indicates the length
testInputs[c] = np.c_[testInputs[c], bias * np.ones((l, 1)),
(l / 30.) * np.ones((l, 1))]
# produce teacher signals. For each input time series of size N x 12 this
# is a time series of size N x 9, all zeros except in the column indicating
# the speaker, where it is 1.
trainOutputs = []
for c in range(270):
l = trainInputs[c].shape[0]
teacher = np.zeros((l, 9))
speakerIndex = int(np.ceil((c + 1) / 30.) - 1)
teacher[:, speakerIndex] = np.ones(l)
trainOutputs.append(teacher)
# produce test output signal
testOutputs = []
speakerIndex = 0
blockCounter = 0
blockLengthes = [31, 35, 88, 44, 29, 24, 40, 50, 29]
for c in range(370):
if blockCounter == blockLengthes[speakerIndex]:
speakerIndex += 1
blockCounter = 0
blockCounter += 1
l = testInputs[c].shape[0]
teacher = np.zeros((l, 9))
teacher[:, speakerIndex] = np.ones(l)
testOutputs.append(teacher)
return trainInputs, trainOutputs, testInputs, testOutputs
#!/usr/bin/env python
from scipy import io
from scipy import stats
from math import sqrt
from decimal import Decimal
file = raw_input('>Give name of bootstrap results file: ')
data = io.read_array(file)
chisq = data[:, 0]
q = data[:, 1]
dphi = data[:, 2]
rd = data[:, 3]
rwd = data[:, 4]
ulimb = data[:, 5]
bsScale = data[:, 6]
bsAz = data[:, 7]
bsFis = data[:, 8]
dExp = data[:, 9]
incl = data[:, 10]
phi0 = data[:, 11]
def printParam(par, parString):
mean = stats.mean(par)
sd = sqrt(stats.var(par))
print parString, ' = ', mean, ' +/- ', sd
printParam(q, 'Q')