def runinversion(basedir,configfile,acfdir='ACF',invtype='tik'):
""" """
costdir = os.path.join(basedir,'Cost')
pname=os.path.join(costdir,'cost{0}-{1}.pickle'.format(acfdir,invtype))
pickleFile = open(pname, 'rb')
alpha_arr=pickle.load(pickleFile)[-1]
pickleFile.close()
ionoinfname=os.path.join(basedir,acfdir,'00lags.h5')
ionoin=IonoContainer.readh5(ionoinfname)
dirio = ('Spectrums','Mat','ACFMat')
inputdir = os.path.join(basedir,dirio[0])
dirlist = glob.glob(os.path.join(inputdir,'*.h5'))
(listorder,timevector,filenumbering,timebeg,time_s) = IonoContainer.gettimes(dirlist)
Ionolist = [dirlist[ikey] for ikey in listorder]
if acfdir.lower()=='acf':
ionosigname=os.path.join(basedir,acfdir,'00sigs.h5')
ionosigin=IonoContainer.readh5(ionosigname)
nl,nt,np1,np2=ionosigin.Param_List.shape
sigs=ionosigin.Param_List.reshape((nl*nt,np1,np2))
sigsmean=sp.nanmean(sigs,axis=0)
sigdiag=sp.diag(sigsmean)
sigsout=sp.power(sigdiag/sigdiag[0],.5).real
alpha_arr=sp.ones_like(alpha_arr)*alpha_arr[0]
acfloc='ACFInv'
elif acfdir.lower()=='acfmat':
mattype='matrix'
acfloc='ACFMatInv'
mattype='sim'
RSTO = RadarSpaceTimeOperator(Ionolist,configfile,timevector,mattype=mattype)
if 'perryplane' in basedir.lower() or 'SimpData':
rbounds=[-500,500]
else:
rbounds=[0,500]
ionoout=invertRSTO(RSTO,ionoin,alpha_list=alpha_arr,invtype=invtype,rbounds=rbounds)[0]
outfile=os.path.join(basedir,acfloc,'00lags{0}.h5'.format(invtype))
ionoout.saveh5(outfile)
if acfdir=='ACF':
lagsDatasum=ionoout.Param_List
# !!! This is done to speed up development
lagsNoisesum=sp.zeros_like(lagsDatasum)
Nlags=lagsDatasum.shape[-1]
pulses_s=RSTO.simparams['Tint']/RSTO.simparams['IPP']
Ctt=makeCovmat(lagsDatasum,lagsNoisesum,pulses_s,Nlags)
outfile=os.path.join(basedir,acfloc,'00sigs{0}.h5'.format(invtype))
ionoout.Param_List=Ctt
ionoout.Param_Names=sp.repeat(ionoout.Param_Names[:,sp.newaxis],Nlags,axis=1)
ionoout.saveh5(outfile)
def plotdata(ionofile_in,ionofile_fit,madfile,time1):
fig1,axmat =plt.subplots(2,2,facecolor='w',figsize=(10,10))
axvec = axmat.flatten()
paramlist = ['ne','te','ti','vo']
paramlisti = ['Ne','Te','Ti','Vi']
paramlistiname = ['$N_e$','$T_e$','$T_i$','$V_i$']
paramunit = ['$m^{-3}$','$^\circ$ K','$^\circ$ K','m/s']
boundlist = [[0.,7e11],[500.,3200.],[500.,2500.],[-500.,500.]]
IonoF = IonoContainer.readh5(ionofile_fit)
IonoI = IonoContainer.readh5(ionofile_in)
gfit = GeoData(readIono,[IonoF,'spherical'])
ginp = GeoData(readIono,[IonoI,'spherical'])
data1 = GeoData(readMad_hdf5,[madfile,['nel','te','ti','vo','dnel','dte','dti','dvo']])
data1.data['ne']=sp.power(10.,data1.data['nel'])
data1.data['dne']=sp.power(10.,data1.data['dnel'])
t1,t2 = data1.timelisting()[340]
handlist = []
for inum,iax in enumerate(axvec):
ploth = rangevsparam(data1,data1.dataloc[0,1:],time1,gkey=paramlist[inum],fig=fig1,ax=iax,it=False)
handlist.append(ploth[0])
ploth = rangevsparam(ginp,ginp.dataloc[0,1:],0,gkey=paramlisti[inum],fig=fig1,ax=iax,it=False)
handlist.append(ploth[0])
ploth = rangevsparam(gfit,gfit.dataloc[0,1:],0,gkey=paramlisti[inum],fig=fig1,ax=iax,it=False)
handlist.append(ploth[0])
iax.set_xlim(boundlist[inum])
iax.set_ylabel('Altitude in km')
iax.set_xlabel(paramlistiname[inum]+' in '+paramunit[inum])
# with error bars
plt.tight_layout()
fig1.suptitle('Comparison Without Error Bars\nPFISR Data Times: {0} to {1}'.format(t1,t2))
plt.subplots_adjust(top=0.9)
plt.figlegend( handlist[:3], ['PFISR', 'SimISR Input','SimISR Fit'], loc = 'lower center', ncol=5, labelspacing=0. )
fig2,axmat2 =plt.subplots(2,2,facecolor='w',figsize=(10,10))
axvec2 = axmat2.flatten()
handlist2 = []
for inum,iax in enumerate(axvec2):
ploth = rangevsparam(data1,data1.dataloc[0,1:],time1,gkey=paramlist[inum],gkeyerr='d'+paramlist[inum],fig=fig2,ax=iax,it=False)
handlist2.append(ploth[0])
ploth = rangevsparam(ginp,ginp.dataloc[0,1:],0,gkey=paramlisti[inum],fig=fig2,ax=iax,it=False)
handlist2.append(ploth[0])
ploth = rangevsparam(gfit,gfit.dataloc[0,1:],0,gkey=paramlisti[inum],gkeyerr='n'+paramlisti[inum],fig=fig2,ax=iax,it=False)
handlist2.append(ploth[0])
iax.set_xlim(boundlist[inum])
iax.set_ylabel('Altitude in km')
iax.set_xlabel(paramlistiname[inum]+' in '+paramunit[inum])
plt.tight_layout()
fig2.suptitle('Comparison With Error Bars\nPFISR Data Times: {0} to {1}'.format(t1,t2))
plt.subplots_adjust(top=0.9)
plt.figlegend( handlist2[:3], ['PFISR', 'SimISR Input','SimISR Fit'], loc = 'lower center', ncol=5, labelspacing=0. )
return (fig1,axvec,handlist,fig2,axvec2,handlist2)
def plotspectra(simfile,realfile,coords = [0.,0.,250.],timesim=0,timereal=345):
fmax=50e3
simiono=IonoContainer.readh5(simfile)
realiono=IonoContainer.readh5(realfile)
t1,t2 = realiono.timelisting()[timereal]
fig1 =plt.figure(facecolor='w',figsize=(10,10))
ax0=plt.subplot2grid((2,2),(0,0))
ax1=plt.subplot2grid((2,2),(0,1))
ax2=plt.subplot2grid((2,2),(1,0),colspan=2)
reald_closest=realiono.getclosest(coords)
real_lags=reald_closest[0][timereal]
sim_closest=simiono.getclosest(coords)
sim_lags=sim_closest[0][timesim]
Nlags = sp.minimum(len(sim_lags),len(real_lags))
l =sp.arange(Nlags)
Nspec=128
simspec = scfft.fftshift(scfft.fft(sim_lags[:Nlags],n=128)).real
realspec= scfft.fftshift(scfft.fft(real_lags[:Nlags],n=128)).real
f=1e-3*sp.arange(-sp.floor(float(Nspec)/2.),sp.ceil(float(Nspec)/2.)) * fmax/Nspec
handlist = []
handlist.append(ax0.plot(l,real_lags[:Nlags].real,'b-',label='Real')[0])
handlist.append(ax0.plot(l,sim_lags[:Nlags].real,'r-',label='Sim')[0])
ax0.set_title('Real Part of ACF')
ax0.set_xlabel('Lag Number')
ax0.set_ylabel('Amplitude')
handlist.append(ax1.plot(l,real_lags[:Nlags].imag,'b-',label='Real')[0])
handlist.append(ax1.plot(l,sim_lags[:Nlags].imag,'r-',label='Sim')[0])
ax1.set_title('Imaginary Part of ACF')
ax1.set_xlabel('Lag Number')
handlist.append(ax2.plot(f,realspec,'b-',label='Real')[0])
handlist.append(ax2.plot(f,simspec,'r-',label='Sim')[0])
ax2.set_title('Spectra')
ax2.set_xlabel('Frequency in kHz')
ax2.set_ylabel('Amplitude')
plt.figlegend( handlist[:2], ['PFISR', 'SimISR'], loc = 'lower center', ncol=5, labelspacing=0. )
fig1.suptitle('Location: [{2} km,{3} km, {4} km] \nPFISR Data Times: {0} to {1}'.format(t1,t2,*coords))
plt.subplots_adjust(top=0.9)
return fig1,handlist
def plotbackground(testdir,figname):
"""
Plots the background densities and temperatures.
"""
sns.set_style('whitegrid')
sns.set_context('notebook')
sns.color_palette("hls", 8)
filelist = glob.glob(os.path.join(testdir,'Origparams','*.h5'))
numlist = [os.path.splitext(os.path.split(x)[-1])[0] for x in filelist]
numdict = {numlist[i]:filelist[i] for i in range(len(filelist))}
ylim = [100.,500.]
slist = sorted(numlist,key=ke)
ifile = numdict[slist[0]]
Iono_in = IonoContainer.readh5(ifile)
numz=sp.sum(Iono_in.Cart_Coords[:,2]==Iono_in.Cart_Coords[0,2])
redcoords=Iono_in.Cart_Coords[numz/2::numz]
z=redcoords[:,2]
begdata = Iono_in.Param_List[numz/2::numz]
allni=begdata[:,0,:-1,0]
allti=begdata[:,0,:-1,1]
T_all = begdata[:,0,:,1]
N_all=begdata[:,0,:,0]
Te = begdata[:,0,-1,-1]
Ne = begdata[:,0,-1,0]
tini=allti*allni
Nisum=allni.sum(-1)
Ti=tini.sum(-1)/Nisum
ncols=2
figsize = (5*ncols,7)
handlist=[]
fig ,axmat= plt.subplots(nrows=1,ncols=ncols,facecolor='w',figsize=figsize,sharey=True)
for i in range(len(Iono_in.Species)):
p1=axmat[0].plot(N_all[:,i],z,label=Iono_in.Species[i],linewidth=lw)[0]
handlist.append(p1)
axmat[0].set_ylabel('Alt in km',fontsize=fs)
axmat[0].set_ylim(ylim)
axmat[0].set_xlabel(r'Number Density in m$^{-3}$',fontsize=fs)
axmat[0].set_title('Ion and Electron Densities',fontsize=fs)
axmat[0].set_xscale('log')
axmat[0].set_xlim([1e2,3e11])
axmat[0].legend(handlist,Iono_in.Species,loc='upper left',fontsize='large')
# handlist2=[]
p2=axmat[1].plot(Te,z,Ti,z,linewidth=lw)
# for i in range(len(Iono_in.Species)):
# p1=axmat[1].plot(T_all[:,i],z,label=Iono_in.Species[i],linewidth=lw)[0]
# handlist2.append(p1)
axmat[1].set_title('Ion and Electron Temperatures',fontsize=fs)
axmat[1].set_xlabel(r'Temperature in $^{\circ}$K',fontsize=fs)
axmat[1].set_xlim([0,2e3])
axmat[1].set_ylim(ylim)
# axmat[1].legend(handlist2,Iono_in.Species,loc='upper left',fontsize='large')
axmat[1].legend(p2,['Te','Ti'],loc='upper left')
fig.savefig(figname,dpi=300)
def fit2geodata(inputfile):
Ionoin = IonoContainer.readh5(inputfile)
keep = ["Ne", "Te", "Ti"]
GD = GeoData(readIono, [Ionoin])
for ikey in GD.data.keys():
if ikey not in keep:
del GD.data[ikey]
indir, fnameall = os.path.split(inputfile)
fname = os.path.splitext(fnameall)[0]
GD.write_h5(os.path.join(indir, fname + "GEOD.h5"))
print("Saved " + os.path.join(indir, fname + ".h5"))
def fixspecs(basedirlist):
for ibase in basedirlist:
filelist = glob.glob(os.path.join(ibase,'Origparams','*.h5'))
numlist = [os.path.splitext(os.path.split(x)[-1])[0] for x in filelist]
numdict = {numlist[i]:filelist[i] for i in range(len(filelist))}
slist = sorted(numlist,key=ke)
origlist = [numdict[i] for i in slist]
filelist = glob.glob(os.path.join(ibase,'Spectrums','*.h5'))
numlist = [os.path.splitext(os.path.split(x)[-1])[0] for x in filelist]
numdict = {numlist[i]:filelist[i] for i in range(len(filelist))}
slist = sorted(numlist,key=ke)
speclist = [numdict[i] for i in slist]
for (iorig,ispec) in zip(origlist,speclist):
origiono=IonoContainer.readh5(iorig)
speciono=IonoContainer.readh5(ispec)
speciono.Cart_Coords=origiono.Cart_Coords
speciono.Sphere_Coords=origiono.Sphere_Coords
os.remove(ispec)
speciono.saveh5(ispec)
def startvalfunc(Ne_init, loc, locsp, time, inputs, ambinfo=[0]):
""" This is a method to determine the start values for the fitter.
Inputs
Ne_init - A nloc x nt numpy array of the initial estimate of electron density. Basically
the zeroth lag of the ACF.
loc - A nloc x 3 numpy array of cartisian coordinates.
time - A nt x 2 numpy array of times in seconds
exinputs - A list of extra inputs allowed for by the fitter class. It only
has one element and its the name of the ionocontainer file holding the
rest of the start parameters.
Outputs
xarray - This is a numpy array of starting values for the fitter parameters."""
if isinstance(inputs, str):
if os.path.splitext(inputs)[-1] == '.h5':
ionoin = IonoContainer.readh5(inputs)
elif os.path.splitext(inputs)[-1] == '.mat':
ionoin = IonoContainer.readmat(inputs)
elif os.path.splitext(inputs)[-1] == '':
ionoin = makeionocombined(inputs)
elif isinstance(inputs, list):
ionoin = makeionocombined(inputs)
else:
ionoin = inputs
numel = sp.prod(ionoin.Param_List.shape[-2:]) + 1
xarray = sp.zeros((loc.shape[0], len(time), numel))
for ilocn, iloc in enumerate(loc):
#for iamb in ambinfo:
newlocsp = locsp[ilocn]
#newlocsp[0] += iamb
(datast, vel) = ionoin.getclosestsphere(newlocsp, time)[:2]
datast[:, -1, 0] = Ne_init[ilocn, :]
# get the ion densities
ionoden = datast[:, :-1, 0]
# find the right normalization for the ion species
ionodensum = sp.repeat(sp.sum(ionoden, axis=-1)[:, sp.newaxis],
ionoden.shape[-1],
axis=-1)
# renormalized to the right level
ionoden = sp.repeat(Ne_init[ilocn, :, sp.newaxis],
ionoden.shape[-1],
axis=-1) * ionoden / ionodensum
datast[:, :-1, 0] = ionoden
xarray[ilocn, :, :-1] = sp.reshape(
datast, (len(time), numel - 1)) #/float(len(ambinfo))
locmag = sp.sqrt(sp.sum(iloc * iloc))
ilocmat = sp.repeat(iloc[sp.newaxis, :], len(time), axis=0)
xarray[ilocn, :,
-1] = sp.sum(vel * ilocmat) / locmag #/float(len(ambinfo))
return xarray
def plotparams(datapath,indch):
# read in the files
geofile = os.path.join(datapath,'Fitted','fitteddataGEOD.h5')
acffile = os.path.join(datapath,'ACF','00lags.h5')
geod = GeoData.read_h5(geofile)
acfIono = IonoContainer.readh5(acffile)
picklefile = os.path.join(datapath,'config.ini')
(sensdict,simparams) = readconfigfile(picklefile)
#determine the ind
dataloc = geod.dataloc
rngloc = 210
ind = sp.argmin(sp.absolute(dataloc[:,0]-rngloc))
#
Ne = geod.data['Ne'][ind]
Ti = geod.data['Ti'][ind]
Te = geod.data['Te'][ind]
#make a spectrum
npts = 128
datablock = sp.array([[Ne[indch],Ti[indch]],[Ne[indch],Te[indch]]])
specobj = ISRSpectrum(centerFrequency =sensdict['fc'],nspec = npts,sampfreq=sensdict['fs'])
(omeg,specexample,rcs) = specobj.getspecsep(datablock,simparams['species'],rcsflag=True)
specexample = rcs*npts*specexample/sp.absolute(specexample).sum()
acf = acfIono.Param_List[ind,indch]
origspec = scfft.fftshift(scfft.fft(acf,n=npts)).real
(figmplf, [[ax1,ax2],[ax3,ax4]]) = plt.subplots(2, 2,figsize=(16, 12), facecolor='w')
ax1.plot(sp.arange(len(Ne)),Ne)
ax1.set_title('$N_e$')
ax2.plot(sp.arange(len(Ti)),Ti)
ax2.set_title('$T_i$')
ax3.plot(sp.arange(len(Te)),Te)
ax3.set_title('$T_e$');
spec1 = ax4.plot(omeg*1e-3,origspec,label='Measured')
spec2 = ax4.plot(omeg*1e-3,specexample,label='Fitted')
ax4.set_title('Measured and Fitted Spectrums')
ax4.set_xlabel('Frequency KHz')
ax4.legend(loc = 1)
# Nevals = sp.linspace(5e10,5e11,10)
# Tivals = sp.linspace(1e3,2e5,10)
# Tevals = sp.linspace(1e3,2e5,10)
# xlist = [[1],Tivals,Nevals,Tevals,[0]]
# outsurf = makefitsurf(xlist,acf,sensdict,simparams)
return(figmplf)
def makespectrums(basedir, configfile, printlines=True):
""" This will make all of the spectra for a set of data and save it in a
folder in basedir called Spectrums. It is assumed that the data in the Origparams
is time tagged in the title with a string seperated by a white space and the
rest of the file name. For example ~/DATA/Basedir/0 origdata.h5.
Inputs:
basedir: A string for the directory that will hold all of the data for the simulation.
configfile: The configuration file for the simulation.
"""
basedir = Path(basedir).expanduser()
dirio = ('Origparams', 'Spectrums')
inputdir = basedir/dirio[0]
outputdir = basedir/dirio[1]
# determine the list of h5 files in Origparams directory
dirlist = sorted(inputdir.glob('*.h5'))
# Make the lists of numbers and file names for the dictionary
(listorder, _, _, timebeg, _) = IonoContainer.gettimes(dirlist)
slist = [dirlist[ikey] for ikey in listorder]
(sensdict, simparams) = readconfigfile(configfile)
# Delete data
outfiles = outputdir.glob('*.h5')
for ifile in outfiles:
ifile.unlink()
for inum, curfile in zip(timebeg, slist):
outfile = outputdir / (str(inum)+' spectrum.h5')
update_progress(float(inum)/float(len(slist)),
'Processing file {} starting at {}'.format(curfile.name, datetime.now()))
curiono = IonoContainer.readh5(str(curfile))
curiono.makespectruminstanceopen(specfuncs.ISRSspecmake, sensdict,
int(simparams['numpoints']), float(inum),
float(len(slist)), printlines).saveh5(str(outfile))
update_progress(float(inum+1)/float(len(slist)),
'Finished file {} starting at {}'.format(curfile.name, datetime.now()))
def plotinputdata(testdir,imgdir,wtimes=False):
"""
This will plot all of the input data with each time step as a pcolor images of
electron density and ion tempreture.
Inputs
testdir - The directory with the input data in h5 files formated for the ionocontainer structure.
imgdir - The directory that holds the images.
"""
if os.path.exists(imgdir):
imgfiles = glob.glob(os.path.join(imgdir,'*.png'))
for imgf in imgfiles:
os.remove(imgf)
else:
os.mkdir(imgdir)
filelist = glob.glob(os.path.join(testdir,'Origparams','*.h5'))
numlist = [os.path.splitext(os.path.split(x)[-1])[0] for x in filelist]
numdict = {numlist[i]:filelist[i] for i in range(len(filelist))}
slist = sorted(numlist,key=ke)
imcount = 0
filetemplate = 'inputdata'
dsetname = os.path.split(os.path.dirname(testdir))[-1]
print "Plotting input data for "+dsetname
if 'perryplane' in testdir.lower():
xlim = [-200.,360.]
xticks = [-150.,0.,150.,300.]
allparams = True
ncols=3
figsize = (15,7)
else:
xlim = [0.,400.]
xticks = [0.,150.,300.]
allparams = False
ncols=1
figsize = (5,7)
ylim = [100.,500.]
f1 = True
for inumn, inum in enumerate(slist):
print "{0} Input for {1} of {2}".format(dsetname,inumn,len(slist))
ifile = numdict[inum]
iono = IonoContainer.readh5(ifile)
Iono1 = GeoData(readIono,[iono])
nt = Iono1.times.shape[0]
rng = sp.sqrt(Iono1.dataloc[:,0]**2+Iono1.dataloc[:,1]**2)*sp.sign(Iono1.dataloc[:,1])
z = Iono1.dataloc[:,2]
rngvec = sp.unique(rng)
zvec = sp.unique(z)
rngmat = rng.reshape(len(zvec),len(rngvec))
zmat = z.reshape(len(zvec),len(rngvec))
Ne = Iono1.data['Ne'].reshape(len(zvec),len(rngvec),nt)*ne_red
Ti = Iono1.data['Ti'].reshape(len(zvec),len(rngvec),nt)
Te = Iono1.data['Te'].reshape(len(zvec),len(rngvec),nt)
for itimen,itime in enumerate(Iono1.times):
if f1:
f1=False
t0 = itime[0]
fig ,axmat= plt.subplots(nrows=1,ncols=ncols,facecolor='w',figsize=figsize ,sharey=True)
if allparams:
avec = axmat.flatten()
else:
avec =[axmat]
plt.sca(avec[0])
plt.xticks(xticks)
plt.tick_params(labelsize=16)
avec[0].set_xlabel('X Plane in km',fontsize=fs)
avec[0].set_ylabel('Alt in km',fontsize=fs)
pc1 = avec[0].pcolor(rngmat,zmat,Ne[:,:,itimen],cmap = defmap,vmin=nemin,vmax=nemax)
avec[0].set_xlim(xlim)
avec[0].set_ylim(ylim)
avec[0].set_title('Electron Density',fontsize=fs)
#pc1.set_norm(colors.LogNorm(vmin=5e8,vmax=5e12))
tick_locator = ticker.MaxNLocator(nbins=5)
cb1 = plt.colorbar(pc1, ax=avec[0])
cb1.ax.xaxis.set_label_position('top')
cb1.ax.set_xlabel(necbarstr,fontsize=fscb)
cb1.locator = tick_locator
cb1.update_ticks()
if allparams:
plt.sca(avec[1])
plt.xticks(xticks)
plt.tick_params(labelsize=16)
avec[1].set_xlabel('X Plane in km',fontsize=fs)
pc2 = avec[1].pcolor(rngmat,zmat,Te[:,:,itimen],cmap = defmap,vmin=temin,vmax=temax)
avec[1].set_xlim(xlim)
avec[1].set_ylim(ylim)
avec[1].set_title('Electron Temperature',fontsize=fs)
cb2 = plt.colorbar(pc2, ax=avec[1])
cb2.ax.xaxis.set_label_position('top')
cb2.ax.set_xlabel(tecbarstr,fontsize=fscb)
cb2.locator = tick_locator
cb2.update_ticks()
plt.sca(avec[2])
plt.xticks(xticks)
plt.tick_params(labelsize=16)
avec[2].set_xlabel('X Plane in km',fontsize=fs)
pc3 = avec[2].pcolor(rngmat,zmat,Ti[:,:,itimen],cmap = defmap,vmin=timin,vmax=timax)
avec[2].set_xlim(xlim)
avec[2].set_ylim(ylim)
avec[2].set_title('Ion Temperature',fontsize=fs)
cb3 = plt.colorbar(pc3, ax=avec[2])
cb3.ax.xaxis.set_label_position('top')
cb3.ax.set_xlabel(ticbarstr,fontsize=fscb)
cb3.locator = tick_locator
cb3.update_ticks()
plt.tight_layout()
if wtimes:
plt.subplots_adjust(top=0.9)
spti = fig.suptitle('Parameters at {0} seconds'.format(int(itime[0]-t0)),fontsize=24)
fname= '{0:0>3}_'.format(imcount)+filetemplate+'.png'
# for ax in avec:
# for label in (ax.get_xticklabels() + ax.get_yticklabels()):
# label.set_fontsize(20)
imcount=imcount+1
plt.savefig(os.path.join(imgdir,fname),dpi=200)
plt.close(fig)
def makehistdata(params, maindir):
""" This will make the histogram data for the statistics.
Inputs
params - A list of parameters that will have statistics created
maindir - The directory that the simulation data is held.
Outputs
datadict - A dictionary with the data values in numpy arrays. The keys are param names.
errordict - A dictionary with the data values in numpy arrays. The keys are param names.
errdictrel - A dictionary with the error values in numpy arrays, normalized by the correct value. The keys are param names.
"""
maindir = Path(maindir)
ffit = maindir.joinpath('Fitted', 'fitteddata.h5')
inputfiledir = maindir.joinpath('Origparams')
paramslower = [ip.lower() for ip in params]
eparamslower = ['n' + ip.lower() for ip in params]
# set up data dictionary
errordict = {ip: [] for ip in params}
errordictrel = {ip: [] for ip in params}
#Read in fitted data
Ionofit = IonoContainer.readh5(str(ffit))
times = Ionofit.Time_Vector
dataloc = Ionofit.Sphere_Coords
rng = dataloc[:, 0]
rng_log = sp.logical_and(rng > 200., rng < 400)
dataloc_out = dataloc[rng_log]
pnames = Ionofit.Param_Names
pnameslower = sp.array([ip.lower() for ip in pnames.flatten()])
p2fit = [
sp.argwhere(ip == pnameslower)[0][0] if ip in pnameslower else None
for ip in paramslower
]
datadict = {
ip: Ionofit.Param_List[rng_log, :, p2fit[ipn]].flatten()
for ipn, ip in enumerate(params)
}
ep2fit = [
sp.argwhere(ip == pnameslower)[0][0] if ip in pnameslower else None
for ip in eparamslower
]
edatadict = {
ip: Ionofit.Param_List[rng_log, :, ep2fit[ipn]].flatten()
for ipn, ip in enumerate(params)
}
# Determine which input files are to be used.
dirlist = [str(i) for i in inputfiledir.glob('*.h5')]
_, outime, filelisting, _, _ = IonoContainer.gettimes(dirlist)
time2files = []
for itn, itime in enumerate(times):
log1 = (outime[:, 0] >= itime[0]) & (outime[:, 0] < itime[1])
log2 = (outime[:, 1] > itime[0]) & (outime[:, 1] <= itime[1])
log3 = (outime[:, 0] <= itime[0]) & (outime[:, 1] > itime[1])
tempindx = sp.where(log1 | log2 | log3)[0]
time2files.append(filelisting[tempindx])
curfilenum = -1
for iparam, pname in enumerate(params):
curparm = paramslower[iparam]
# Use Ne from input to compare the ne derived from the power.
if curparm == 'nepow':
curparm = 'ne'
datalist = []
for itn, itime in enumerate(times):
for filenum in time2files[itn]:
filenum = int(filenum)
if curfilenum != filenum:
curfilenum = filenum
datafilename = dirlist[filenum]
Ionoin = IonoContainer.readh5(datafilename)
if ('ti' in paramslower) or ('vi' in paramslower):
Ionoin = maketi(Ionoin)
pnames = Ionoin.Param_Names
pnameslowerin = sp.array(
[ip.lower() for ip in pnames.flatten()])
prmloc = sp.argwhere(curparm == pnameslowerin)
if prmloc.size != 0:
curprm = prmloc[0][0]
# build up parameter vector bs the range values by finding the closest point in space in the input
curdata = sp.zeros(len(dataloc_out))
for irngn, curcoord in enumerate(dataloc_out):
tempin = Ionoin.getclosestsphere(curcoord, [itime])[0]
Ntloc = tempin.shape[0]
tempin = sp.reshape(tempin, (Ntloc, len(pnameslowerin)))
curdata[irngn] = tempin[0, curprm]
datalist.append(curdata)
errordict[pname] = datadict[pname] - sp.hstack(datalist)
errordictrel[pname] = 100. * errordict[pname] / sp.absolute(
sp.hstack(datalist))
return datadict, errordict, errordictrel, edatadict
def parametersweep(basedir,configfile,acfdir='ACF',invtype='tik'):
"""
This function will run the inversion numerious times with different constraint
parameters. This will create a directory called cost and place.
Input
basedir - The directory that holds all of the data for the simulator.
configfile - The ini file for the simulation.
acfdir - The directory within basedir that hold the acfs to be inverted.
invtype - The inversion method that will be tested. Can be tik, tikd, and tv.
"""
alpha_sweep=sp.logspace(-3.5,sp.log10(7),25)
costdir = os.path.join(basedir,'Cost')
ionoinfname=os.path.join(basedir,acfdir,'00lags.h5')
ionoin=IonoContainer.readh5(ionoinfname)
dirio = ('Spectrums','Mat','ACFMat')
inputdir = os.path.join(basedir,dirio[0])
dirlist = glob.glob(os.path.join(inputdir,'*.h5'))
(listorder,timevector,filenumbering,timebeg,time_s) = IonoContainer.gettimes(dirlist)
Ionolist = [dirlist[ikey] for ikey in listorder]
RSTO = RadarSpaceTimeOperator(Ionolist,configfile,timevector,mattype='Sim')
npts=RSTO.simparams['numpoints']
ionospec=makeionocombined(dirlist)
if npts==ionospec.Param_List.shape[-1]:
tau,acfin=spect2acf(ionospec.Param_Names,ionospec.Param_List)
nloc,ntimes=acfin.shape[:2]
ambmat=RSTO.simparams['amb_dict']['WttMatrix']
np=ambmat.shape[0]
acfin_amb=sp.zeros((nloc,ntimes,np),dtype=acfin.dtype)
# get the original acf
ambmat=RSTO.simparams['amb_dict']['WttMatrix']
np=ambmat.shape[0]
for iloc,locarr in enumerate(acfin):
for itime,acfarr in enumerate(locarr):
acfin_amb[iloc,itime]=sp.dot(ambmat,acfarr)
acfin_amb=acfin_amb[:,0]
else:
acfin_amb=ionospec.Param_List[:,0]
if not os.path.isdir(costdir):
os.mkdir(costdir)
# pickle file stuff
pname=os.path.join(costdir,'cost{0}-{1}.pickle'.format(acfdir,invtype))
alpha_list=[]
errorlist=[]
errorlaglist=[]
datadiflist=[]
constlist=[]
if 'perryplane' in basedir.lower() or 'SimpData':
rbounds=[-500,500]
else:
rbounds=[0,500]
alpha_list_new=alpha_sweep.tolist()
for i in alpha_list:
if i in alpha_list_new:
alpha_list_new.remove(i)
for i in alpha_list_new:
ionoout,datadif,constdif=invertRSTO(RSTO,ionoin,alpha_list=i,invtype=invtype,rbounds=rbounds,Nlin=1)
datadiflist.append(datadif)
constlist.append(constdif)
acfout=ionoout.Param_List[:,0]
alpha_list.append(i)
outdata=sp.power(sp.absolute(acfout-acfin_amb),2)
aveerror=sp.sqrt(sp.nanmean(outdata,axis=0))
errorlaglist.append(aveerror)
errorlist.append(sp.nansum(aveerror))
pickleFile = open(pname, 'wb')
pickle.dump([alpha_list,errorlist,datadiflist,constlist,errorlaglist],pickleFile)
pickleFile.close()
mkalphalist(pname)
alphaarr=sp.array(alpha_list)
errorarr=sp.array(errorlist)
errorlagarr=sp.array(errorlaglist)
datadif=sp.array(datadiflist)
constdif=sp.array(constlist)
fig,axlist,axmain=plotalphaerror(alphaarr,errorarr,errorlagarr)
fig.savefig(os.path.join(costdir,'cost{0}-{1}.png'.format(acfdir,invtype)))
fig,axlist=plotLcurve(alphaarr,datadif,constdif)
fig.savefig(os.path.join(costdir,'lcurve{0}-{1}.png'.format(acfdir,invtype)))
def makehistdata(params,maindir):
""" This will make the histogram data for the statistics.
Inputs
params - A list of parameters that will have statistics created
maindir - The directory that the simulation data is held.
Outputs
datadict - A dictionary with the data values in numpy arrays. The keys are param names.
errordict - A dictionary with the data values in numpy arrays. The keys are param names.
errdictrel - A dictionary with the error values in numpy arrays, normalized by the correct value. The keys are param names.
"""
maindir=Path(maindir)
ffit = maindir.joinpath('Fitted','fitteddata.h5')
inputfiledir = maindir.joinpath('Origparams')
paramslower = [ip.lower() for ip in params]
eparamslower = ['n'+ip.lower() for ip in params]
# set up data dictionary
errordict = {ip:[] for ip in params}
errordictrel = {ip:[] for ip in params}
#Read in fitted data
Ionofit = IonoContainer.readh5(ffit)
times=Ionofit.Time_Vector
dataloc = Ionofit.Sphere_Coords
pnames = Ionofit.Param_Names
pnameslower = sp.array([ip.lower() for ip in pnames.flatten()])
p2fit = [sp.argwhere(ip==pnameslower)[0][0] if ip in pnameslower else None for ip in paramslower]
datadict = {ip:Ionofit.Param_List[:,:,p2fit[ipn]].flatten() for ipn, ip in enumerate(params)}
ep2fit = [sp.argwhere(ip==pnameslower)[0][0] if ip in pnameslower else None for ip in eparamslower]
edatadict = {ip:Ionofit.Param_List[:,:,ep2fit[ipn]].flatten() for ipn, ip in enumerate(params)}
# Determine which input files are to be used.
dirlist = [str(i) for i in inputfiledir.glob('*.h5')]
sortlist,outime,filelisting,timebeg,timelist_s = IonoContainer.gettimes(dirlist)
time2files = []
for itn,itime in enumerate(times):
log1 = (outime[:,0]>=itime[0]) & (outime[:,0]<itime[1])
log2 = (outime[:,1]>itime[0]) & (outime[:,1]<=itime[1])
log3 = (outime[:,0]<=itime[0]) & (outime[:,1]>itime[1])
tempindx = sp.where(log1|log2|log3)[0]
time2files.append(filelisting[tempindx])
curfilenum=-1
for iparam,pname in enumerate(params):
curparm = paramslower[iparam]
# Use Ne from input to compare the ne derived from the power.
if curparm == 'nepow':
curparm = 'ne'
datalist = []
for itn,itime in enumerate(times):
for iplot,filenum in enumerate(time2files[itn]):
filenum = int(filenum)
if curfilenum!=filenum:
curfilenum=filenum
datafilename = dirlist[filenum]
Ionoin = IonoContainer.readh5(datafilename)
if ('ti' in paramslower) or ('vi' in paramslower):
Ionoin = maketi(Ionoin)
pnames = Ionoin.Param_Names
pnameslowerin = sp.array([ip.lower() for ip in pnames.flatten()])
prmloc = sp.argwhere(curparm==pnameslowerin)
if prmloc.size !=0:
curprm = prmloc[0][0]
# build up parameter vector bs the range values by finding the closest point in space in the input
curdata = sp.zeros(len(dataloc))
for irngn, curcoord in enumerate(dataloc):
tempin = Ionoin.getclosestsphere(curcoord,[itime])[0]
Ntloc = tempin.shape[0]
tempin = sp.reshape(tempin,(Ntloc,len(pnameslowerin)))
curdata[irngn] = tempin[0,curprm]
datalist.append(curdata)
errordict[pname] = datadict[pname]-sp.hstack(datalist)
errordictrel[pname] = 100.*errordict[pname]/sp.absolute(sp.hstack(datalist))
return datadict,errordict,errordictrel,edatadict
def plotsampling(testdir,outfile, ifile=None, wtimes=False):
"""This will plot all of the input data with each time step as a pcolor images of
electron density and ion temperature.
Inputs
testdir - The directory with the input data in h5 files formated for the ionocontainer structure.
imgdir - The directory that holds the images."""
imcount = 0
if ifile is None:
filelist = glob.glob(os.path.join(testdir,'Origparams','*.h5'))
numlist = [os.path.splitext(os.path.split(x)[-1])[0] for x in filelist]
numdict = {numlist[i]:filelist[i] for i in range(len(filelist))}
slist = sorted(numlist,key=ke)
inumn=0
inum = slist[0]
dsetname = os.path.split(os.path.dirname(testdir))[-1]
print "Plotting input data for "+dsetname
print "{0} Input for {1} of {2}".format(dsetname,inumn,len(slist))
ifile = numdict[inum]
iono = IonoContainer.readh5(ifile)
filetemplate = 'inputdata'
if 'perryplane' in testdir.lower():
xlim = [-150.,360.]
xticks = [0.,150.,300.]
allparams = True
ncols=3
figsize = (5,7)
else:
xlim = [0.,400.]
xticks = [0.,150.,300.]
allparams = False
ncols=1
figsize = (5,7)
ylim = [100.,500.]
f1 = True
if iono.Param_List.shape[-1]==1:
nemax=1e10*ne_red
nemin=0
iono2 = IonoContainer.readh5(outfile)
xout,yout,zout = iono2.Cart_Coords.transpose()
rout=sp.sqrt(xout**2+yout**2)
Iono1 = GeoData(readIono,[iono])
nt = Iono1.times.shape[0]
rng = sp.sqrt(Iono1.dataloc[:,0]**2+Iono1.dataloc[:,1]**2)*sp.sign(Iono1.dataloc[:,1]).astype('float32')
z = Iono1.dataloc[:,2].astype('float32')
rngvec = sp.unique(rng)
zvec = sp.unique(z)
rngmat = rng.reshape(len(zvec),len(rngvec))
zmat = z.reshape(len(zvec),len(rngvec))
if 'Ne' in Iono1.data.keys():
Ne = Iono1.data['Ne'].reshape(len(zvec),len(rngvec),nt)*ne_red
elif '0' in Iono1.data.keys():
Ne = Iono1.data['Ne'].reshape(len(zvec),len(rngvec),nt)*ne_red
itimen=0
itime = Iono1.times[0]
if f1:
f1=False
t0 = itime[0]
fig ,axmat= plt.subplots(nrows=1,ncols=ncols,facecolor='w',figsize=figsize ,sharey=True)
if allparams:
avec = axmat.flatten()
else:
avec =[axmat]
plt.sca(avec[0])
plt.xticks(xticks)
plt.tick_params(labelsize=16)
avec[0].set_xlabel('X Plane in km',fontsize=fs)
avec[0].set_ylabel('Alt in km',fontsize=fs)
pc1 = avec[0].pcolor(rngmat,zmat,Ne[:,:,itimen],cmap = defmap,vmin=nemin,vmax=nemax)
avec[0].set_xlim(xlim)
avec[0].set_ylim(ylim)
#pc1.set_norm(colors.LogNorm(vmin=5e8,vmax=5e12))
tick_locator = ticker.MaxNLocator(nbins=5)
cb1 = plt.colorbar(pc1, ax=avec[0])
cb1.ax.xaxis.set_label_position('top')
cb1.ax.set_xlabel(necbarstr,fontsize=fscb)
cb1.locator = tick_locator
cb1.update_ticks()
plot1 = avec[0].plot(rout,zout,'w.',markersize=3)
plt.tight_layout()
if wtimes:
plt.subplots_adjust(top=0.9)
spti = fig.suptitle('Parameters at {0} seconds'.format(int(itime[0]-t0)),fontsize=24)
return (fig,avec)
def plotgradlines(inputlist,fitiono,alt,times,paramlist=['Ne','Te','Ti']):
"""
Plots the values along a specific alittude.
"""
inputiono = makeionocombined(inputlist)
Iono1 = GeoData(readIono,[inputiono])
fitiono = IonoContainer.readh5(fitiono)
fitGeo = GeoData(readIono,[fitiono])
paramlist = ['Ne','Te','Ti']
(x,y,z) = inputiono.Cart_Coords.transpose()
r = sp.sqrt(x**2+y**2)*sp.sign(y)
incoords = sp.column_stack((r,z,sp.ones_like(z)))
ru,zu =[ sp.unique(r),sp.unique(z)]
Rmat,Zmat = sp.meshgrid(ru,zu)
zinput = sp.argmin(sp.absolute(zu-alt))
(xf,yf,zf) = fitiono.Cart_Coords.transpose()
rf = sp.sqrt(xf**2+yf**2)*sp.sign(yf)
outcoords = sp.column_stack((rf,zf,sp.ones_like(zf)))
rfu,zfu =[ sp.unique(rf),sp.unique(zf)]
Rfmat,Zfmat = sp.meshgrid(rfu,zfu)
zoutput = sp.argmin(sp.absolute(zfu-alt))
fitGeo.interpolate(incoords,Iono1.coordnames,method='linear',fill_value=sp.nan,twodinterp = True,oldcoords=outcoords)
uz = sp.unique(z)
ur = sp.unique(r)
(rmat,zmat) = sp.meshgrid(ur,uz)
inputdata = {}
for iparm in paramlist:
if iparm =='Nepow':
iparm='Ne'
curdata = Iono1.data[iparm][:,times[0]]
inputdata[iparm] = sp.reshape(curdata,Rmat.shape)[zinput]
outputdata = {}
for iparm in paramlist:
curdata = fitGeo.data[iparm][:, times[1]]
outputdata[iparm] = sp.reshape(curdata,Rmat.shape)[zinput]
fig, axvec = plt.subplots(len(paramlist),1,sharey=False,figsize=(12,5*len(paramlist)))
for ipn,iparam in enumerate(paramlist):
ax = axvec[ipn]
ax2 = ax.twinx()
p1, = ax.plot(ru,inputdata[iparam],'b-',label='In',linewidth=3)
p2, = ax.plot(ru,outputdata[iparam],'b--',label='Out',linewidth=3)
ax.set_title(iparam)
ax.set_xlabel('X Plane in km')
gi = sp.gradient(inputdata[iparam])/sp.gradient(ru)
go = sp.gradient(outputdata[iparam])/sp.gradient(ru)
p3, = ax2.plot(ru,gi,'g-',label='Grad In',linewidth=3)
p4, = ax2.plot(ru,go,'g--',label='Grad Out',linewidth=3)
ax.yaxis.label.set_color(p1.get_color())
ax2.yaxis.label.set_color(p3.get_color())
lines = [p1,p2,p3,p4]
ax.legend(lines,[l.get_label() for l in lines])
plt.tight_layout()
return(fig)
def fitdata(basedir,configfile,optinputs):
""" This function will run the fitter on the estimated ACFs saved in h5 files.
Inputs:
basedir: A string for the directory that will hold all of the data for the simulation.
configfile: The configuration file for the simulation.
optinputs:A string that helps determine the what type of acfs will be fitted.
"""
# determine the input folders which can be ACFs from the full simulation
dirdict = {'fitting':('ACF', 'Fitted'), 'fittingmat':('ACFMat', 'FittedMat'),
'fittinginv':('ACFInv', 'FittedInv'), 'fittingmatinv':('ACFMatInv', 'FittedMatInv')}
dirio = dirdict[optinputs[0]]
inputdir = basedir/dirio[0]
outputdir = basedir/dirio[1]
fitlist = optinputs[1]
if len(optinputs) > 2:
exstr = optinputs[2]
printlines = optinputs[3]
else:
exstr = ''
dirlist = [str(i) for i in inputdir.glob('*lags{0}.h5'.format(exstr))]
dirlistsig = [str(i) for i in inputdir.glob('*sigs{0}.h5'.format(exstr))]
Ionoin = IonoContainer.readh5(dirlist[0])
if len(dirlistsig) == 0:
Ionoinsig = None
else:
Ionoinsig = IonoContainer.readh5(dirlistsig[0])
fitterone = Fitterionoconainer(Ionoin, Ionoinsig, configfile)
fitoutput = fitterone.fitdata(specfuncs.ISRSfitfunction,
fitterone.simparams['startfile'], fittimes=fitlist,
printlines=printlines)
#ipdb.set_trace()
(fitteddata, fittederror, funcevals, fittedcov) = fitoutput
if fitterone.simparams['Pulsetype'].lower() == 'barker':
paramlist = fitteddata
species = fitterone.simparams['species']
paramnames = ['Ne']
if not fittederror is None:
fittederronly = sp.sqrt(fittederror)
paramlist = sp.concatenate([fitteddata, fittederronly], axis=2)
paramnamese = ['n'+ip for ip in paramnames]
paranamsf = sp.array(paramnames+paramnamese)
else:
paranamsf = sp.array(paramnames)
else:
fittederronly = sp.sqrt(fittederror)
paramnames = []
species = fitterone.simparams['species']
# Seperate Ti and put it in as an element of the ionocontainer.
Ti = fitteddata[:, :, 1]
nTi = fittederronly[:, :, 1]
nTiTe = fittedcov[:, :, 0, 1]
nTiNe = fittedcov[:, :, 0, 2]
nTiVi = fittedcov[:, :, 0, 3]
nTeNe = fittedcov[:, :, 1, 2]
nTeVi = fittedcov[:, :, 1, 3]
nNeVi = fittedcov[:, :, 2, 3]
cov_list = [nTiTe[:, :, sp.newaxis], nTiNe[:, :, sp.newaxis],
nTiVi[:, :, sp.newaxis], nTeNe[:, :, sp.newaxis],
nTeVi[:, :, sp.newaxis], nNeVi[:, :, sp.newaxis]]
cov_list_names = ['nTiTe', 'nTiNe', 'nTiVi', 'nTeNe', 'nTeVi','nNeVi']
paramlist = sp.concatenate([fitteddata, Ti[:, :, sp.newaxis], fittederronly,
nTi[:, :, sp.newaxis], funcevals[:, :, sp.newaxis]]
+ cov_list, axis=2)
for isp in species[:-1]:
paramnames.append('Ni_'+isp)
paramnames.append('Ti_'+isp)
paramnames = paramnames+['Ne', 'Te', 'Vi', 'Nepow', 'Ti']
paramnamese = ['n'+ip for ip in paramnames]
paranamsf = sp.array(paramnames+paramnamese+['FuncEvals']+cov_list_names)
if fitlist is None:
timevec = Ionoin.Time_Vector
else:
if len(fitlist) == 0:
timevec = Ionoin.Time_Vector
else:
timevec = Ionoin.Time_Vector[fitlist]
# This requires
if set(Ionoin.Coord_Vecs) == {'x', 'y', 'z'}:
newver = 0
ionoout = IonoContainer(Ionoin.Cart_Coords, paramlist.real, timevec, ver=newver,
coordvecs=Ionoin.Coord_Vecs, paramnames=paranamsf,
species=species)
elif set(Ionoin.Coord_Vecs) == {'r', 'theta', 'phi'}:
newver = 1
ionoout = IonoContainer(Ionoin.Sphere_Coords, paramlist.real, timevec, ver=newver,
coordvecs=Ionoin.Coord_Vecs, paramnames=paranamsf,
species=species)
outfile = outputdir.joinpath('fitteddata{0}.h5'.format(exstr))
ionoout.saveh5(str(outfile))
def plotacf(testdir,imgdir,wtimes=False,acfpath='ACFInv',lagfile='00lags.h5',filetemplate=None,lag=0):
"""
This will plot all of the fitted data with each time step as a pcolor images of
electron density and electron density from power mesurements.
Inputs
testdir - The directory with the input data in h5 files formated for the ionocontainer structure.
imgdir - The directory that holds the images.
"""
# if os.path.exists(imgdir):
# imgfiles = glob.glob(os.path.join(imgdir,'*.png'))
# for imgf in imgfiles:
# os.remove(imgf)
if not os.path.exists(imgdir):
os.mkdir(imgdir)
filename = os.path.join(testdir,acfpath,lagfile)
iono = IonoContainer.readh5(filename)
# if iono.Param_List.shape[-1]==1:
# nemax=1e10
# nemin=0
nt = iono.Time_Vector.shape[0]
if set(iono.Coord_Vecs)=={'r','theta','phi'}:
rngrdr =iono.Sphere_Coords[:,0].astype('float32')
sign1 = sp.sign(iono.Sphere_Coords[:,1])
el = iono.Sphere_Coords[:,2].astype('float32')
elvec,elinv = sp.unique(el,return_inverse=True)
nbeams = len(elvec)
nrg = len(rngrdr)/nbeams
Rngrdrmat = sp.reshape(rngrdr,(nrg,nbeams))
Signmat = sp.reshape(sign1,(nrg,nbeams))
Elmat = sp.reshape(el,(nrg,nbeams))
Xmat = Rngrdrmat*Signmat*sp.cos(Elmat*sp.pi/180.)
Zmat = Rngrdrmat*sp.sin(Elmat*sp.pi/180.)
Ne = iono.Param_List[:,:,lag].reshape(nrg,nbeams,nt).real*ne_red
elif set(iono.Coord_Vecs)=={'x','y','z'}:
rng = sp.sqrt(iono.Cart_Coords[:,0]**2+iono.Cart_Coords[:,1]**2)*sp.sign(iono.Cart_Coords[:,1]).astype('float32')
z = iono.Cart_Coords[:,2].astype('float32')
rngvec = sp.unique(rng)
zvec = sp.unique(z)
Xmat = rng.reshape(len(zvec),len(rngvec))
Zmat = z.reshape(len(zvec),len(rngvec))
Ne = iono.Param_List[:,:,lag].reshape(len(zvec),len(rngvec),nt).real*ne_red
imcount=0
if filetemplate is None:
filetemplate = 'acflag{0}'.format(lag)
dsetname = os.path.split(os.path.dirname(testdir))[-1]
print "Plotting Output data for "+dsetname
if 'perryplane' in testdir.lower():
xlim = [-200.,360.]
xticks = [-150.,0.,150.,300.]
else:
xlim = [0.,400.]
xticks = [0.,150.,300]
ncols=1
figsize = (5,7)
ylim = [100.,500]
for itimen,itime in enumerate(iono.Time_Vector):
print "{0} Output for {1} of {2}".format(dsetname,itimen,len(iono.Time_Vector))
Nemat = Ne[:,:,itimen]
fig ,axmat= plt.subplots(nrows=1,ncols=ncols,facecolor='w',figsize=figsize,sharey=True)
avec=[axmat]
plt.sca(avec[0])
avec[0].set_xlabel('X Plane in km',fontsize=fs)
avec[0].set_ylabel('Alt in km',fontsize=fs)
pc1 = avec[0].pcolor(Xmat,Zmat,Nemat,cmap = defmap,vmin=nemin,vmax=nemax)
plt.tick_params(labelsize=16)
plt.xticks(xticks)
avec[0].set_xlim(xlim)
avec[0].set_ylim(ylim)
avec[0].set_title('',fontsize=fs)
tick_locator = ticker.MaxNLocator(nbins=5)
cb1 = plt.colorbar(pc1, ax=avec[0])
cb1.ax.xaxis.set_label_position('top')
cb1.ax.set_xlabel(necbarstr)
cb1.locator = tick_locator
cb1.update_ticks()
plt.tight_layout()
fname= '{0:0>3}_'.format(imcount)+filetemplate+'.png'
plt.savefig(os.path.join(imgdir,fname),dpi=300)
imcount=imcount+1
plt.close(fig)
def plotpercenterror(testdir,imgdir,config,wtimes=False,fitpath='Fitted',fitfile='fitteddata.h5'):
"""
This will plot all of the fitted data with each time step as a pcolor images of
electron density and electron density from power mesurements.
Inputs
testdir - The directory with the input data in h5 files formated for the ionocontainer structure.
imgdir - The directory that holds the images.
"""
(sensdict,simparams)=readconfigfile(config)
tvec = simparams['Timevec']
filetemplate = 'fitteddataerrorpercent'
if os.path.exists(imgdir):
imgfiles = glob.glob(os.path.join(imgdir,'*'+filetemplate+'.png'))
for imgf in imgfiles:
os.remove(imgf)
else:
os.mkdir(imgdir)
filename = os.path.join(testdir,fitpath,fitfile)
iono = IonoContainer.readh5(filename)
Iono1 = GeoData(readIono,[iono])
rngrdr =Iono1.dataloc[:,0].astype('float32')
sign1 = sp.sign(Iono1.dataloc[:,1])
el = Iono1.dataloc[:,2].astype('float32')
elvec,elinv = sp.unique(el,return_inverse=True)
nbeams = len(elvec)
nrg = len(rngrdr)/nbeams
nt = Iono1.times.shape[0]
Rngrdrmat = sp.reshape(rngrdr,(nrg,nbeams))
Signmat = sp.reshape(sign1,(nrg,nbeams))
Elmat = sp.reshape(el,(nrg,nbeams))
permin,permax=[0.,100.]
Xmat = Rngrdrmat*Signmat*sp.cos(Elmat*sp.pi/180.)
Zmat = Rngrdrmat*sp.sin(Elmat*sp.pi/180.)
Ne = Iono1.data['Ne'].reshape(nrg,nbeams,nt)
Ti = Iono1.data['Ti'].reshape(nrg,nbeams,nt)
Te = Iono1.data['Te'].reshape(nrg,nbeams,nt)
nNe = 100.*Iono1.data['nNe'].reshape(nrg,nbeams,nt)/Ne
nTe = 100.*Iono1.data['nTe'].reshape(nrg,nbeams,nt)/Te
nTi = 100.*Iono1.data['nTi'].reshape(nrg,nbeams,nt)/Ti
imcount=0
dsetname = os.path.split(os.path.dirname(testdir))[-1]
print "Plotting Output error data for "+dsetname
if 'perryplane' in testdir.lower():
xlim = [-200.,360.]
xticks = [-150.,0.,150.,300.]
allparams=True
ncols=3
figsize = (15,7)
else:
xlim = [0.,400.]
xticks = [0.,150.,300]
allparams = False
ncols=1
figsize = (5,7)
ylim = [100.,500]
for itimen,itime in enumerate(Iono1.times):
print "{0} Output for {1} of {2}".format(dsetname,itimen,len(Iono1.times))
Nemat = nNe[:,:,itimen]
Timat = nTi[:,:,itimen]
Temat = nTe[:,:,itimen]
fig ,axmat= plt.subplots(nrows=1,ncols=ncols,facecolor='w',figsize=figsize,sharey=True)
if allparams:
avec=axmat.flatten()
else:
avec=[axmat]
plt.sca(avec[0])
avec[0].set_xlabel('X Plane in km',fontsize=fs)
avec[0].set_ylabel('Alt in km',fontsize=fs)
pc1 = avec[0].pcolor(Xmat,Zmat,Nemat,cmap = defmap,vmin=permin,vmax=permax)
plt.tick_params(labelsize=16)
plt.xticks(xticks)
avec[0].set_xlim(xlim)
avec[0].set_ylim(ylim)
avec[0].set_title('Electron Density % Error',fontsize=fs)
tick_locator = ticker.MaxNLocator(nbins=5)
cb1 = plt.colorbar(pc1, ax=avec[0])
cb1.ax.xaxis.set_label_position('top')
cb1.ax.set_xlabel(ercbarstr,fontsize=fscb)
cb1.locator = tick_locator
cb1.update_ticks()
if allparams:
plt.sca(avec[1])
plt.tick_params(labelsize=16)
plt.xticks(xticks)
avec[1].set_xlabel('X Plane in km',fontsize=fs)
pc2 = avec[1].pcolor(Xmat,Zmat,Temat,cmap = defmap,vmin=permin,vmax=permax)
avec[1].set_xlim(xlim)
avec[1].set_ylim(ylim)
avec[1].set_title('Electron Temperature % Error',fontsize=fs)
cb2 = plt.colorbar(pc2, ax=avec[1])
cb2.ax.xaxis.set_label_position('top')
cb2.ax.set_xlabel(ercbarstr,fontsize=fscb)
cb2.locator = tick_locator
cb2.update_ticks()
plt.sca(avec[2])
plt.xticks(xticks)
plt.tick_params(labelsize=16)
avec[2].set_xlabel('X Plane in km',fontsize=fs)
pc3 = avec[2].pcolor(Xmat,Zmat,Timat,cmap = defmap,vmin=permin,vmax=permax)
avec[2].set_xlim(xlim)
avec[2].set_ylim(ylim)
avec[2].set_title('Ion Temperature % Error',fontsize=fs)
cb3 = plt.colorbar(pc3, ax=avec[2])
cb3.ax.xaxis.set_label_position('top')
cb3.ax.set_xlabel(ercbarstr,fontsize=fscb)
cb3.locator = tick_locator
cb3.update_ticks()
plt.tight_layout()
if wtimes:
plt.subplots_adjust(top=0.9)
spti = fig.suptitle('Percent Error at {0} seconds'.format(int(tvec[itimen])),fontsize=24)
fname= '{0:0>3}_'.format(imcount)+filetemplate+'.png'
plt.savefig(os.path.join(imgdir,fname),dpi=300)
imcount=imcount+1
plt.close(fig)
def __init__(self, Ionodict, inifile, outdir, outfilelist=None):
"""
This function will create an instance of the RadarData class. It will
take in the values and create the class and make raw IQ data.
Inputs:
sensdict - A dictionary of sensor parameters
angles - A list of tuples which the first position is the az angle
and the second position is the el angle.
IPP - The interpulse period in seconds represented as a float.
Tint - The integration time in seconds as a float. This will be the
integration time of all of the beams.
time_lim - The length of time of the simulation the number of time points
will be calculated.
pulse - A numpy array that represents the pulse shape.
rng_lims - A numpy array of length 2 that holds the min and max range
that the radar will cover.
"""
(sensdict, simparams) = readconfigfile(inifile)
self.simparams = simparams
N_angles = len(self.simparams['angles'])
NNs = int(self.simparams['NNs'])
self.sensdict = sensdict
Npall = sp.floor(self.simparams['TimeLim']/self.simparams['IPP'])
Npall = int(sp.floor(Npall/N_angles)*N_angles)
Np = Npall/N_angles
print("All spectrums created already")
filetimes = Ionodict.keys()
filetimes.sort()
ftimes = sp.array(filetimes)
simdtype = self.simparams['dtype']
pulsetimes = sp.arange(Npall)*self.simparams['IPP'] +ftimes.min()
pulsefile = sp.array([sp.where(itimes-ftimes >= 0)[0][-1] for itimes in pulsetimes])
# differentiate between phased arrays and dish antennas
if sensdict['Name'].lower() in ['risr', 'pfisr', 'risr-n']:
beams = sp.tile(sp.arange(N_angles), Npall/N_angles)
else:
# for dish arrays
brate = simparams['beamrate']
beams2 = sp.repeat(sp.arange(N_angles), brate)
beam3 = sp.concatenate((beams2, beams2[::-1]))
ntile = int(sp.ceil(Npall/len(beam3)))
leftover = int(Npall-ntile*len(beam3))
if ntile > 0:
beams = sp.tile(beam3, ntile)
beams = sp.concatenate((beams, beam3[:leftover]))
else:
beams = beam3[:leftover]
pulsen = sp.repeat(sp.arange(Np), N_angles)
pt_list = []
pb_list = []
pn_list = []
fname_list = []
self.datadir = outdir
self.maindir = outdir.parent
self.procdir = self.maindir/'ACF'
Nf = len(filetimes)
progstr = 'Data from {:d} of {:d} being processed Name: {:s}.'
if outfilelist is None:
print('\nData Now being created.')
Noisepwr = v_Boltz*sensdict['Tsys']*sensdict['BandWidth']
self.outfilelist = []
for ifn, ifilet in enumerate(filetimes):
outdict = {}
ifile = Ionodict[ifilet]
ifilename = Path(ifile).name
update_progress(float(ifn)/Nf, progstr.format(ifn, Nf, ifilename))
curcontainer = IonoContainer.readh5(ifile)
if ifn == 0:
self.timeoffset = curcontainer.Time_Vector[0, 0]
pnts = pulsefile == ifn
pt = pulsetimes[pnts]
pb = beams[pnts]
pn = pulsen[pnts].astype(int)
rawdata = self.__makeTime__(pt, curcontainer.Time_Vector,
curcontainer.Sphere_Coords,
curcontainer.Param_List, pb)
d_shape = rawdata.shape
n_tempr = sp.random.randn(*d_shape).astype(simdtype)
n_tempi = 1j*sp.random.randn(*d_shape).astype(simdtype)
noise = sp.sqrt(Noisepwr/2)*(n_tempr+n_tempi)
outdict['AddedNoise'] = noise
outdict['RawData'] = rawdata+noise
outdict['RawDatanonoise'] = rawdata
outdict['NoiseData'] = sp.sqrt(Noisepwr/2)*(sp.random.randn(len(pn), NNs).astype(simdtype)+
1j*sp.random.randn(len(pn), NNs).astype(simdtype))
outdict['Pulses'] = pn
outdict['Beams'] = pb
outdict['Time'] = pt
fname = '{0:d} RawData.h5'.format(ifn)
newfn = self.datadir/fname
self.outfilelist.append(str(newfn))
dict2h5(str(newfn), outdict)
#Listing info
pt_list.append(pt)
pb_list.append(pb)
pn_list.append(pn)
fname_list.append(fname)
infodict = {'Files':fname_list, 'Time':pt_list, 'Beams':pb_list, 'Pulses':pn_list}
dict2h5(str(outdir.joinpath('INFO.h5')), infodict)
else:
infodict = h52dict(str(outdir.joinpath('INFO.h5')))
alltime = sp.hstack(infodict['Time'])
self.timeoffset = alltime.min()
self.outfilelist = outfilelist
def __init__(self, Ionodict, inifile, outdir, outfilelist=None):
"""
This function will create an instance of the RadarData class. It will
take in the values and create the class and make raw IQ data.
Inputs:
sensdict - A dictionary of sensor parameters
angles - A list of tuples which the first position is the az angle
and the second position is the el angle.
IPP - The interpulse period in seconds represented as a float.
Tint - The integration time in seconds as a float. This will be the
integration time of all of the beams.
time_lim - The length of time of the simulation the number of time points
will be calculated.
pulse - A numpy array that represents the pulse shape.
rng_lims - A numpy array of length 2 that holds the min and max range
that the radar will cover.
"""
(sensdict, simparams) = readconfigfile(inifile)
self.simparams = simparams
N_angles = len(self.simparams['angles'])
NNs = int(self.simparams['NNs'])
self.sensdict = sensdict
Npall = sp.floor(self.simparams['TimeLim'] / self.simparams['IPP'])
Npall = int(sp.floor(Npall / N_angles) * N_angles)
Np = Npall / N_angles
print("All spectrums created already")
filetimes = Ionodict.keys()
filetimes.sort()
ftimes = sp.array(filetimes)
simdtype = self.simparams['dtype']
pulsetimes = sp.arange(Npall) * self.simparams['IPP'] + ftimes.min()
pulsefile = sp.array(
[sp.where(itimes - ftimes >= 0)[0][-1] for itimes in pulsetimes])
# differentiate between phased arrays and dish antennas
if sensdict['Name'].lower() in ['risr', 'pfisr', 'risr-n']:
beams = sp.tile(sp.arange(N_angles), Npall / N_angles)
else:
# for dish arrays
brate = simparams['beamrate']
beams2 = sp.repeat(sp.arange(N_angles), brate)
beam3 = sp.concatenate((beams2, beams2[::-1]))
ntile = int(sp.ceil(Npall / len(beam3)))
leftover = int(Npall - ntile * len(beam3))
if ntile > 0:
beams = sp.tile(beam3, ntile)
beams = sp.concatenate((beams, beam3[:leftover]))
else:
beams = beam3[:leftover]
pulsen = sp.repeat(sp.arange(Np), N_angles)
pt_list = []
pb_list = []
pn_list = []
fname_list = []
self.datadir = outdir
self.maindir = outdir.parent
self.procdir = self.maindir / 'ACF'
Nf = len(filetimes)
progstr = 'Data from {:d} of {:d} being processed Name: {:s}.'
if outfilelist is None:
print('\nData Now being created.')
Noisepwr = v_Boltz * sensdict['Tsys'] * sensdict['BandWidth']
self.outfilelist = []
for ifn, ifilet in enumerate(filetimes):
outdict = {}
ifile = Ionodict[ifilet]
ifilename = Path(ifile).name
update_progress(
float(ifn) / Nf, progstr.format(ifn, Nf, ifilename))
curcontainer = IonoContainer.readh5(ifile)
if ifn == 0:
self.timeoffset = curcontainer.Time_Vector[0, 0]
pnts = pulsefile == ifn
pt = pulsetimes[pnts]
pb = beams[pnts]
pn = pulsen[pnts].astype(int)
rawdata = self.__makeTime__(pt, curcontainer.Time_Vector,
curcontainer.Sphere_Coords,
curcontainer.Param_List, pb)
d_shape = rawdata.shape
n_tempr = sp.random.randn(*d_shape).astype(simdtype)
n_tempi = 1j * sp.random.randn(*d_shape).astype(simdtype)
noise = sp.sqrt(Noisepwr / 2) * (n_tempr + n_tempi)
outdict['AddedNoise'] = noise
outdict['RawData'] = rawdata + noise
outdict['RawDatanonoise'] = rawdata
outdict['NoiseData'] = sp.sqrt(Noisepwr / 2) * (
sp.random.randn(len(pn), NNs).astype(simdtype) +
1j * sp.random.randn(len(pn), NNs).astype(simdtype))
outdict['Pulses'] = pn
outdict['Beams'] = pb
outdict['Time'] = pt
fname = '{0:d} RawData.h5'.format(ifn)
newfn = self.datadir / fname
self.outfilelist.append(str(newfn))
dict2h5(str(newfn), outdict)
#Listing info
pt_list.append(pt)
pb_list.append(pb)
pn_list.append(pn)
fname_list.append(fname)
infodict = {
'Files': fname_list,
'Time': pt_list,
'Beams': pb_list,
'Pulses': pn_list
}
dict2h5(str(outdir.joinpath('INFO.h5')), infodict)
else:
infodict = h52dict(str(outdir.joinpath('INFO.h5')))
alltime = sp.hstack(infodict['Time'])
self.timeoffset = alltime.min()
self.outfilelist = outfilelist
