def runfitter(paramdict, SNR, n_pulses, n_runs, Niratio, x_0=SVALS):
"""
paramdict
"""
data = SIMVALUES
z = sp.linspace(50., 1e3, 50)
nz = len(z)
params = sp.tile(data[sp.newaxis, sp.newaxis, :, :], (nz, 1, 1, 1))
coords = sp.column_stack((sp.ones(nz), sp.ones(nz), z))
species = ['O+', 'e-']
times = sp.array([[0, 1e9]])
vel = sp.zeros((nz, 1, 3))
(sensdict, simparams) = readconfigfile(defcon)
species = paramdict['species']
nspecies = len(species)
ni = nspecies-1
Icont1 = IonoContainer(coordlist=coords, paramlist=params, times=times,
sensor_loc=sp.zeros(3), ver=0, coordvecs=['x', 'y', 'z'],
paramnames=None, species=species, velocity=vel)
omeg, outspecs = specfuncs.ISRSspecmake(Icont1, sensdict, int(simparams['numpoints']))
tau, acf = spect2acf(omeg, outspecs)
t_log = sp.logical_and(tau<simparams['Pulselength'],tau>=0)
plen = t_log.sum()
amb_dict = simparams['amb_dict']
amb_mat_flat = sp.zeros_like(amb_dict['WttMatrix'])
eyemat = sp.eye(plen)
amb_mat_flat[:plen, t_log] = eyemat
fitmode = simparams['fitmode']
simparams['amb_dict']['WttMatrix'] = amb_mat_flat
Nloc, Nt = acf.shape[:2]
# output Files
fittedarray = sp.zeros((Nloc, Nt, nparams+1))*sp.nan
fittederror = sp.zeros((Nloc, Nt, nparams+1))*sp.nan
fittedcov = sp.zeros((Nloc, Nt, 4, 4))*sp.nan
funcevals = sp.zeros((Nloc, Nt))
for iloc, ilocvec in acf:
for itn, iacf in ilocvec:
curlag = sp.dot(amb_mat_flat, iacf)
std_lag = sp.absolute(curlag)(1.+1./SNR)/sp.sqrt(n_pulses)
covar_lag = sp.diag(sp.power(std_lag,2.))
curlag = curlag + std_lag*sp.random.randn(curlag.shape)
d_func = (curlag, sensdict, simparams, Niratio)
# Only fit Ti, Te, Ne and Vi
# change variables because of new fit mode
# Perform the fitting
optresults = scipy.optimize.least_squares(fun=specfuncs.ISRSfitfunction, x0=x_0,
method='lm', verbose=0, args=d_func)
x_res = optresults.x.real
# Derive data for the ions using output from the fitter and ion
# species ratios which are assumed to be given.
ionstuff = sp.zeros(ni*2-1)
ionstuff[:2*ni:2] = x_res[1]*Niratio
ionstuff[1:2*ni-1:2] = x_res[0]
# change variables because of new fit mode
if fitmode == 1:
x_res[2] = x_res[2]*x_res[0]
fittedarray[iloc, itn] = sp.append(ionstuff,
sp.append(x_res, Ne_start[iloc, itime]))
funcevals[iloc, itn] = optresults.nfev
# fittedarray[iloc,itime] = sp.append(optresults.x,Ne_start[iloc,itime])
resid = optresults.cost
jac = optresults.jac
# combine the rows because of the complex conjugates
jacc = jac[0::2]+jac[1::2]
try:
# Derive covariances for the ions using output from the fitter and ion species ratios which are assumed to be given.
#covf = sp.linalg.inv(sp.dot(jac.transpose(),jac))*resid/dof
covf = sp.linalg.inv(sp.dot(sp.dot(jacc.transpose(),
sp.linalg.inv(covar_lag)), jacc))
# change variables because of new fit mode
if fitmode == 1:
# is this right?
covf[2] = covf[2]*x_res[0]
covf[:,2] = covf[:,2]*x_res[0]
vars_vec = sp.diag(covf).real
ionstuff = sp.zeros(ni*2-1)
ionstuff[:2*ni:2] = vars_vec[1]*Niratio
ionstuff[1:2*ni-1:2] = vars_vec[0]
vars_vec = sp.append(ionstuff, vars_vec)
fittedcov[iloc, itn] = covf
except:#sp.linalg.LinAlgError('singular matrix'):
vars_vec = sp.ones(nparams)*float('nan')
# if len(vars_vec)<fittederror.shape[-1]-1:
# pdb.set_trace()
fittederror[iloc, itn, :-1] = vars_vec
return(fittedarray, fittederror, funcevals, fittedcov)
def mult_iono(self,ionoin_list):
"""
This will apply the forward model to the contents of an ionocontainer object. It is assuming that
this is an ionocontainer holding the spectra.
"""
ntout = self.Time_Out.shape[0]
nlout = self.Cart_Coords_Out.shape[0]
blist_in,blist_out = self.blocklocs
amb_dict = self.simparams['amb_dict']
ambmat = amb_dict['WttMatrix']
overlaps = self.overlaps
t_s = self.sensdict['t_s']
if isinstance(ionoin_list,list)or isinstance(ionoin_list,str):
Iono_in = makeionocombined(ionoin_list)
else:
Iono_in=ionoin_list
ionocart = Iono_in.Cart_Coords
if self.simparams['numpoints']==Iono_in.Param_List.shape[-1]:
tau,acf=spect2acf(Iono_in.Param_Names,Iono_in.Param_List)
np = ambmat.shape[0]
else:
acf=Iono_in.Param_List
np = acf.shape[-1]
np_in =acf.shape[-1]
tau_out = t_s*np.arange(np)
outdata = np.zeros((nlout,ntout,np),dtype=acf.dtype)
assert np.allclose(ionocart,self.Cart_Coords_In), "Spatial Coordinates need to be the same"
for it_out in range(ntout):
overlists = overlaps[it_out]
irows = blist_out[it_out]
curintimes = [i[0] for i in overlists]
curintratio=[i[1] for i in overlists]
if self.mattype.lower()=='sim':
curintimes=[curintimes[0]]
curintratio=[1.]
cur_outmat = self.RSTMat[irows[0]:irows[1],:]
icols= blist_in[it_out]
cur_mat = cur_outmat[:,icols[0]:icols[1]]
for i_it,it_in in enumerate(curintimes):
tempdata=np.zeros((np_in,nlout),dtype=acf.dtype)
for iparam in range(np_in):
tempdata[iparam]=cur_mat.dot(acf[:,it_in,iparam])
if self.simparams['numpoints']==Iono_in.Param_List.shape[-1]:
tempdata=np.dot(ambmat,tempdata)
outdata[:,it_out] = np.transpose(tempdata)*curintratio[i_it] + outdata[:,it_out]
outiono = IonoContainer(self.Sphere_Coords_Out,outdata,times=self.Time_Out,sensor_loc=Iono_in.Sensor_loc,
ver=1,coordvecs = ['r','theta','phi'],paramnames=tau_out)
return outiono
def parametersweep(basedir,configfile,acfdir='ACF',invtype='tik'):
"""
This function will run the inversion numerious times with different constraint
parameters. This will create a directory called cost and place.
Input
basedir - The directory that holds all of the data for the simulator.
configfile - The ini file for the simulation.
acfdir - The directory within basedir that hold the acfs to be inverted.
invtype - The inversion method that will be tested. Can be tik, tikd, and tv.
"""
alpha_sweep=sp.logspace(-3.5,sp.log10(7),25)
costdir = os.path.join(basedir,'Cost')
ionoinfname=os.path.join(basedir,acfdir,'00lags.h5')
ionoin=IonoContainer.readh5(ionoinfname)
dirio = ('Spectrums','Mat','ACFMat')
inputdir = os.path.join(basedir,dirio[0])
dirlist = glob.glob(os.path.join(inputdir,'*.h5'))
(listorder,timevector,filenumbering,timebeg,time_s) = IonoContainer.gettimes(dirlist)
Ionolist = [dirlist[ikey] for ikey in listorder]
RSTO = RadarSpaceTimeOperator(Ionolist,configfile,timevector,mattype='Sim')
npts=RSTO.simparams['numpoints']
ionospec=makeionocombined(dirlist)
if npts==ionospec.Param_List.shape[-1]:
tau,acfin=spect2acf(ionospec.Param_Names,ionospec.Param_List)
nloc,ntimes=acfin.shape[:2]
ambmat=RSTO.simparams['amb_dict']['WttMatrix']
np=ambmat.shape[0]
acfin_amb=sp.zeros((nloc,ntimes,np),dtype=acfin.dtype)
# get the original acf
ambmat=RSTO.simparams['amb_dict']['WttMatrix']
np=ambmat.shape[0]
for iloc,locarr in enumerate(acfin):
for itime,acfarr in enumerate(locarr):
acfin_amb[iloc,itime]=sp.dot(ambmat,acfarr)
acfin_amb=acfin_amb[:,0]
else:
acfin_amb=ionospec.Param_List[:,0]
if not os.path.isdir(costdir):
os.mkdir(costdir)
# pickle file stuff
pname=os.path.join(costdir,'cost{0}-{1}.pickle'.format(acfdir,invtype))
alpha_list=[]
errorlist=[]
errorlaglist=[]
datadiflist=[]
constlist=[]
if 'perryplane' in basedir.lower() or 'SimpData':
rbounds=[-500,500]
else:
rbounds=[0,500]
alpha_list_new=alpha_sweep.tolist()
for i in alpha_list:
if i in alpha_list_new:
alpha_list_new.remove(i)
for i in alpha_list_new:
ionoout,datadif,constdif=invertRSTO(RSTO,ionoin,alpha_list=i,invtype=invtype,rbounds=rbounds,Nlin=1)
datadiflist.append(datadif)
constlist.append(constdif)
acfout=ionoout.Param_List[:,0]
alpha_list.append(i)
outdata=sp.power(sp.absolute(acfout-acfin_amb),2)
aveerror=sp.sqrt(sp.nanmean(outdata,axis=0))
errorlaglist.append(aveerror)
errorlist.append(sp.nansum(aveerror))
pickleFile = open(pname, 'wb')
pickle.dump([alpha_list,errorlist,datadiflist,constlist,errorlaglist],pickleFile)
pickleFile.close()
mkalphalist(pname)
alphaarr=sp.array(alpha_list)
errorarr=sp.array(errorlist)
errorlagarr=sp.array(errorlaglist)
datadif=sp.array(datadiflist)
constdif=sp.array(constlist)
fig,axlist,axmain=plotalphaerror(alphaarr,errorarr,errorlagarr)
fig.savefig(os.path.join(costdir,'cost{0}-{1}.png'.format(acfdir,invtype)))
fig,axlist=plotLcurve(alphaarr,datadif,constdif)
fig.savefig(os.path.join(costdir,'lcurve{0}-{1}.png'.format(acfdir,invtype)))
def ISRSfitfunction(x, y_acf, sensdict, simparams, Niratios, y_err=None):
"""
This is the fit fucntion that is used with scipy.optimize.leastsquares. It will
take a set parameter values construct a spectrum/acf based on those values, apply
the ambiguity function and take the difference between the two. Since the ACFs are
complex the arrays split up and the size doubled as it is output.
Inputs
x - A Np array of parameter values used
y_acf - This is the esitmated ACF/spectrum represented as a complex numpy array
sensdict - This is a dictionary that holds many of the sensor parameters.
simparams - This is a dictionary that holds info on the simulation parameters.
y_err - default None - A numpy array of size Nd that holds the standard deviations of the data.
Output
y_diff - A Nd or 2Nd array if input data is complex that is the difference
between the data and the fitted model"""
npts = simparams['numpoints']
specs = simparams['species']
amb_dict = simparams['amb_dict']
numtype = simparams['dtype']
if 'FitType' in simparams.keys():
fitspec = simparams['FitType']
else:
fitspec = 'Spectrum'
nspecs = len(specs)
(Ti, Ne, Te, v_i) = x
datablock = np.zeros((nspecs, 2), dtype=x.dtype)
datablock[:-1, 0] = Ne * Niratios
datablock[:-1, 1] = Ti
datablock[-1, 0] = Ne
datablock[-1, 1] = Te
# determine if you've gone beyond the bounds
# penalty for being less then zero
grt0 = np.exp(-datablock)
pentsum = np.zeros(grt0.size + 1)
pentsum[:-1] = grt0.flatten()
specobj = ISRSpectrum(centerFrequency=sensdict['fc'],
nspec=npts,
sampfreq=sensdict['fs'])
(omeg, cur_spec, rcs) = specobj.getspecsep(datablock,
specs,
v_i,
rcsflag=True)
cur_spec.astype(numtype)
# Create spectrum guess
(tau, acf) = spect2acf(omeg, cur_spec)
if amb_dict['WttMatrix'].shape[-1] != acf.shape[0]:
pdb.set_trace()
guess_acf = np.dot(amb_dict['WttMatrix'], acf)
# apply ambiguity function
guess_acf = guess_acf * rcs / guess_acf[0].real
if fitspec.lower() == 'spectrum':
# fit to spectrums
spec_interm = scfft.fft(guess_acf, n=len(cur_spec))
spec_final = spec_interm.real
y_interm = scfft.fft(y_acf, n=len(spec_final))
y = y_interm.real
yout = (y - spec_final)
elif fitspec.lower() == 'acf':
yout = y_acf - guess_acf
if y_err is not None:
yout = yout * 1. / y_err
# Cannot make the output a complex array! To avoid this problem simply double
# the size of the array and place the real and imaginary parts in alternating spots.
if np.iscomplexobj(yout):
youttmp = yout.copy()
yout = np.zeros(2 * len(youttmp)).astype(youttmp.real.dtype)
yout[::2] = youttmp.real
yout[1::2] = youttmp.imag
penadd = np.sqrt(np.power(np.absolute(yout), 2).sum()) * pentsum.sum()
return yout + penadd
def ISRSfitfunction(x, y_acf, sensdict, simparams, Niratios, y_err=None ):
"""
This is the fit fucntion that is used with scipy.optimize.leastsquares. It will
take a set parameter values construct a spectrum/acf based on those values, apply
the ambiguity function and take the difference between the two. Since the ACFs are
complex the arrays split up and the size doubled as it is output.
Inputs
x - A Np array of parameter values used
y_acf - This is the esitmated ACF/spectrum represented as a complex numpy array
sensdict - This is a dictionary that holds many of the sensor parameters.
simparams - This is a dictionary that holds info on the simulation parameters.
y_err - default None - A numpy array of size Nd that holds the standard deviations of the data.
fitmethod - default 0 - A number representing the input parameters
Output
y_diff - A Nd or 2Nd array if input data is complex that is the difference
between the data and the fitted model"""
npts = simparams['numpoints']
specs = simparams['species']
amb_dict = simparams['amb_dict']
numtype = simparams['dtype']
if 'FitType' in simparams.keys():
fitspec = simparams['FitType']
else:
fitspec = 'Spectrum'
nspecs = len(specs)
if not 'fitmode' in simparams.keys():
(Ti, Ne, Te, v_i) = x
elif simparams['fitmode'] == 0:
(Ti, Ne, Te, v_i) = x
elif simparams['fitmode'] == 1:
(Ti, Ne, TeoTi, v_i) = x
Te = TeoTi*Ti
elif simparams['fitmode'] == 2:
(Ti, acfnorm, TeoTi, v_i) = x
Te = TeoTi*Ti
Ne = acfnorm*(1+TeoTi)
datablock = np.zeros((nspecs, 2), dtype=x.dtype)
datablock[:-1, 0] = Ne*Niratios
datablock[:-1, 1] = Ti
datablock[-1, 0] = Ne
datablock[-1, 1] = Te
# determine if you've gone beyond the bounds
# penalty for being less then zero
grt0 = np.exp(-datablock)
pentsum = np.zeros(grt0.size+1)
pentsum[:-1] = grt0.flatten()
specobj = ISRSpectrum(centerFrequency=sensdict['fc'], nspec=npts, sampfreq=sensdict['fs'])
(omeg, cur_spec, rcs) = specobj.getspecsep(datablock, specs, v_i, rcsflag=True)
cur_spec.astype(numtype)
# Create spectrum guess
(_, acf) = spect2acf(omeg, cur_spec)
if amb_dict['WttMatrix'].shape[-1] != acf.shape[0]:
pdb.set_trace()
guess_acf = np.dot(amb_dict['WttMatrix'], acf)
# apply ambiguity function
guess_acf = guess_acf*rcs/guess_acf[0].real
if fitspec.lower() == 'spectrum':
# fit to spectrums
spec_interm = scfft.fft(guess_acf, n=len(cur_spec))
spec_final = spec_interm.real
y_interm = scfft.fft(y_acf, n=len(spec_final))
y_spec = y_interm.real
yout = y_spec-spec_final
elif fitspec.lower() == 'acf':
yout = y_acf-guess_acf
if y_err is not None:
yout = yout*1./y_err
# Cannot make the output a complex array! To avoid this problem simply double
# the size of the array and place the real and imaginary parts in alternating spots.
if np.iscomplexobj(yout):
youttmp = yout.copy()
yout = np.zeros(2*len(youttmp)).astype(youttmp.real.dtype)
yout[::2] = youttmp.real
yout[1::2] = youttmp.imag
penadd = np.sqrt(np.power(np.absolute(yout), 2).sum())*pentsum.sum()
return yout+penadd
def mult_iono(self, ionoin_list):
"""
This will apply the forward model to the contents of an ionocontainer object. It is assuming that
this is an ionocontainer holding the spectra.
"""
ntout = self.Time_Out.shape[0]
nlout = self.Cart_Coords_Out.shape[0]
blist_in, blist_out = self.blocklocs
amb_dict = self.simparams['amb_dict']
ambmat = amb_dict['WttMatrix']
overlaps = self.overlaps
t_s = self.sensdict['t_s']
if isinstance(ionoin_list, list) or isinstance(ionoin_list, str):
Iono_in = makeionocombined(ionoin_list)
else:
Iono_in = ionoin_list
ionocart = Iono_in.Cart_Coords
if self.simparams['numpoints'] == Iono_in.Param_List.shape[-1]:
tau, acf = spect2acf(Iono_in.Param_Names, Iono_in.Param_List)
np = ambmat.shape[0]
else:
acf = Iono_in.Param_List
np = acf.shape[-1]
np_in = acf.shape[-1]
tau_out = t_s * np.arange(np)
outdata = np.zeros((nlout, ntout, np), dtype=acf.dtype)
assert np.allclose(
ionocart,
self.Cart_Coords_In), "Spatial Coordinates need to be the same"
for it_out in range(ntout):
overlists = overlaps[it_out]
irows = blist_out[it_out]
curintimes = [i[0] for i in overlists]
curintratio = [i[1] for i in overlists]
if self.mattype.lower() == 'sim':
curintimes = [curintimes[0]]
curintratio = [1.]
cur_outmat = self.RSTMat[irows[0]:irows[1], :]
icols = blist_in[it_out]
cur_mat = cur_outmat[:, icols[0]:icols[1]]
for i_it, it_in in enumerate(curintimes):
tempdata = np.zeros((np_in, nlout), dtype=acf.dtype)
for iparam in range(np_in):
tempdata[iparam] = cur_mat.dot(acf[:, it_in, iparam])
if self.simparams['numpoints'] == Iono_in.Param_List.shape[-1]:
tempdata = np.dot(ambmat, tempdata)
outdata[:, it_out] = np.transpose(
tempdata) * curintratio[i_it] + outdata[:, it_out]
outiono = IonoContainer(self.Sphere_Coords_Out,
outdata,
times=self.Time_Out,
sensor_loc=Iono_in.Sensor_loc,
ver=1,
coordvecs=['r', 'theta', 'phi'],
paramnames=tau_out)
return outiono
def main():
(sensdict,simparams) = readconfigfile(inifile)
simdtype = simparams['dtype']
sumrule = simparams['SUMRULE']
npts = simparams['numpoints']
amb_dict = simparams['amb_dict']
# for spectrum
ISS2 = ISRSpectrum(centerFrequency = 440.2*1e6, bMag = 0.4e-4, nspec=npts, sampfreq=sensdict['fs'],dFlag=True)
ti = 2e3
te = 2e3
Ne = 1e11
Ni = 1e11
datablock90 = sp.array([[Ni,ti],[Ne,te]])
species = simparams['species']
(omega,specorig,rcs) = ISS2.getspecsep(datablock90, species,rcsflag = True)
cur_filt = sp.sqrt(scfft.ifftshift(specorig*npts*npts*rcs/specorig.sum()))
#for data
Nrep = 10000
pulse = sp.ones(14)
lp_pnts = len(pulse)
N_samps = 100
minrg = -sumrule[0].min()
maxrg = N_samps+lp_pnts-sumrule[1].max()
Nrng2 = maxrg-minrg;
out_data = sp.zeros((Nrep,N_samps+lp_pnts),dtype=simdtype)
samp_num = sp.arange(lp_pnts)
for isamp in range(N_samps):
cur_pnts = samp_num+isamp
cur_pulse_data = MakePulseDataRep(pulse,cur_filt,rep=Nrep)
out_data[:,cur_pnts] = cur_pulse_data+out_data[:,cur_pnts]
lagsData = CenteredLagProduct(out_data,numtype=simdtype,pulse =pulse)
lagsData=lagsData/Nrep # divide out the number of pulses summed
Nlags = lagsData.shape[-1]
lagsDatasum = sp.zeros((Nrng2,Nlags),dtype=lagsData.dtype)
for irngnew,irng in enumerate(sp.arange(minrg,maxrg)):
for ilag in range(Nlags):
lagsDatasum[irngnew,ilag] = lagsData[irng+sumrule[0,ilag]:irng+sumrule[1,ilag]+1,ilag].mean(axis=0)
lagsDatasum=lagsDatasum/lp_pnts # divide out the pulse length
(tau,acf) = spect2acf(omega,specorig)
# apply ambiguity function
tauint = amb_dict['Delay']
acfinterp = sp.zeros(len(tauint),dtype=simdtype)
acfinterp.real =spinterp.interp1d(tau,acf.real,bounds_error=0)(tauint)
acfinterp.imag =spinterp.interp1d(tau,acf.imag,bounds_error=0)(tauint)
# Apply the lag ambiguity function to the data
guess_acf = sp.zeros(amb_dict['Wlag'].shape[0],dtype=sp.complex128)
for i in range(amb_dict['Wlag'].shape[0]):
guess_acf[i] = sp.sum(acfinterp*amb_dict['Wlag'][i])
# pdb.set_trace()
guess_acf = guess_acf*rcs/guess_acf[0].real
# fit to spectrums
spec_interm = scfft.fftshift(scfft.fft(guess_acf,n=npts))
spec_final = spec_interm.real
allspecs = scfft.fftshift(scfft.fft(lagsDatasum,n=len(spec_final),axis=-1),axes=-1)
# allspecs = scfft.fftshift(scfft.fft(lagsDatasum,n=npts,axis=-1),axes=-1)
fig = plt.figure()
plt.plot(omega,spec_final.real,label='In',linewidth=5)
plt.hold(True)
plt.plot(omega,allspecs[40].real,label='Out',linewidth=5)
plt.axis((omega.min(),omega.max(),0.0,2e11))
plt.show(False)