def makeinputh5(Iono,basedir):
"""This will make a h5 file for the IonoContainer that can be used as starting
points for the fitter. The ionocontainer taken will be average over the x and y dimensions
of space to make an average value of the parameters for each altitude.
Inputs
Iono - An instance of the Ionocontainer class that will be averaged over so it can
be used for fitter starting points.
basdir - A string that holds the directory that the file will be saved to.
"""
# Get the parameters from the original data
Param_List = Iono.Param_List
dataloc = Iono.Cart_Coords
times = Iono.Time_Vector
velocity = Iono.Velocity
zlist,idx = sp.unique(dataloc[:,2],return_inverse=True)
siz = list(Param_List.shape[1:])
vsiz = list(velocity.shape[1:])
datalocsave = sp.column_stack((sp.zeros_like(zlist),sp.zeros_like(zlist),zlist))
outdata = sp.zeros([len(zlist)]+siz)
outvel = sp.zeros([len(zlist)]+vsiz)
# Do the averaging across space
for izn,iz in enumerate(zlist):
arr = sp.argwhere(idx==izn)
outdata[izn]=sp.mean(Param_List[arr],axis=0)
outvel[izn]=sp.mean(velocity[arr],axis=0)
Ionoout = IonoContainer(datalocsave,outdata,times,Iono.Sensor_loc,ver=0,
paramnames=Iono.Param_Names, species=Iono.Species,velocity=outvel)
Ionoout.saveh5(str(basedir/'startdata.h5'))
def makeinputh5(Iono,basedir):
basedir = Path(basedir).expanduser()
Param_List = Iono.Param_List
dataloc = Iono.Cart_Coords
times = Iono.Time_Vector
velocity = Iono.Velocity
zlist,idx = sp.unique(dataloc[:,2],return_inverse=True)
siz = list(Param_List.shape[1:])
vsiz = list(velocity.shape[1:])
datalocsave = sp.column_stack((sp.zeros_like(zlist),sp.zeros_like(zlist),zlist))
outdata = sp.zeros([len(zlist)]+siz)
outvel = sp.zeros([len(zlist)]+vsiz)
for izn,iz in enumerate(zlist):
arr = sp.argwhere(idx==izn)
outdata[izn]=sp.mean(Param_List[arr],axis=0)
outvel[izn]=sp.mean(velocity[arr],axis=0)
Ionoout = IonoContainer(datalocsave,outdata,times,Iono.Sensor_loc,ver=0,
paramnames=Iono.Param_Names, species=Iono.Species,velocity=outvel)
ofn = basedir/'startdata.h5'
print('writing {}'.format(ofn))
Ionoout.saveh5(str(ofn))
def makeinputh5(Iono, basedir):
basedir = Path(basedir).expanduser()
Param_List = Iono.Param_List
dataloc = Iono.Cart_Coords
times = Iono.Time_Vector
velocity = Iono.Velocity
zlist, idx = sp.unique(dataloc[:, 2], return_inverse=True)
siz = list(Param_List.shape[1:])
vsiz = list(velocity.shape[1:])
datalocsave = sp.column_stack(
(sp.zeros_like(zlist), sp.zeros_like(zlist), zlist))
outdata = sp.zeros([len(zlist)] + siz)
outvel = sp.zeros([len(zlist)] + vsiz)
for izn, iz in enumerate(zlist):
arr = sp.argwhere(idx == izn)
outdata[izn] = sp.mean(Param_List[arr], axis=0)
outvel[izn] = sp.mean(velocity[arr], axis=0)
Ionoout = IonoContainer(datalocsave,
outdata,
times,
Iono.Sensor_loc,
ver=0,
paramnames=Iono.Param_Names,
species=Iono.Species,
velocity=outvel)
ofn = basedir / 'startdata.h5'
print('writing {}'.format(ofn))
Ionoout.saveh5(str(ofn))
def makeinputh5(Iono,basedir):
"""This will make a h5 file for the IonoContainer that can be used as starting
points for the fitter. The ionocontainer taken will be average over the x and y dimensions
of space to make an average value of the parameters for each altitude.
Inputs
Iono - An instance of the Ionocontainer class that will be averaged over so it can
be used for fitter starting points.
basdir - A string that holds the directory that the file will be saved to.
"""
# Get the parameters from the original data
Param_List = Iono.Param_List
dataloc = Iono.Cart_Coords
times = Iono.Time_Vector
velocity = Iono.Velocity
zlist,idx = sp.unique(dataloc[:,2],return_inverse=True)
siz = list(Param_List.shape[1:])
vsiz = list(velocity.shape[1:])
datalocsave = sp.column_stack((sp.zeros_like(zlist),sp.zeros_like(zlist),zlist))
outdata = sp.zeros([len(zlist)]+siz)
outvel = sp.zeros([len(zlist)]+vsiz)
# Do the averaging across space
for izn,iz in enumerate(zlist):
arr = sp.argwhere(idx==izn)
outdata[izn] = sp.mean(Param_List[arr],axis=0)
outvel[izn] = sp.mean(velocity[arr],axis=0)
Ionoout = IonoContainer(datalocsave,outdata,times,Iono.Sensor_loc,ver=0,
paramnames=Iono.Param_Names, species=Iono.Species,velocity=outvel)
Ionoout.saveh5(basedir/'startdata.h5')
def makestartfile(datapath):
outdata = sp.array([[[[1e11,3e3],[1e11,3e3]]]])
datalocs = sp.array([[225.0,15.0,89.0]])
pnames = sp.array([['Ni','Ti'],['Ne','Te']])
species=['O2+','e-']
vel = sp.array([[[0,0,0]]])
Ionoout = IonoContainer(datalocs,outdata,sp.array([0]),ver=1,
paramnames=pnames, species=species,velocity=vel)
Ionoout.saveh5(os.path.join(datapath,'startdata.h5'))
def runinversion(basedir,configfile,acfdir='ACF',invtype='tik'):
""" """
costdir = os.path.join(basedir,'Cost')
pname=os.path.join(costdir,'cost{0}-{1}.pickle'.format(acfdir,invtype))
pickleFile = open(pname, 'rb')
alpha_arr=pickle.load(pickleFile)[-1]
pickleFile.close()
ionoinfname=os.path.join(basedir,acfdir,'00lags.h5')
ionoin=IonoContainer.readh5(ionoinfname)
dirio = ('Spectrums','Mat','ACFMat')
inputdir = os.path.join(basedir,dirio[0])
dirlist = glob.glob(os.path.join(inputdir,'*.h5'))
(listorder,timevector,filenumbering,timebeg,time_s) = IonoContainer.gettimes(dirlist)
Ionolist = [dirlist[ikey] for ikey in listorder]
if acfdir.lower()=='acf':
ionosigname=os.path.join(basedir,acfdir,'00sigs.h5')
ionosigin=IonoContainer.readh5(ionosigname)
nl,nt,np1,np2=ionosigin.Param_List.shape
sigs=ionosigin.Param_List.reshape((nl*nt,np1,np2))
sigsmean=sp.nanmean(sigs,axis=0)
sigdiag=sp.diag(sigsmean)
sigsout=sp.power(sigdiag/sigdiag[0],.5).real
alpha_arr=sp.ones_like(alpha_arr)*alpha_arr[0]
acfloc='ACFInv'
elif acfdir.lower()=='acfmat':
mattype='matrix'
acfloc='ACFMatInv'
mattype='sim'
RSTO = RadarSpaceTimeOperator(Ionolist,configfile,timevector,mattype=mattype)
if 'perryplane' in basedir.lower() or 'SimpData':
rbounds=[-500,500]
else:
rbounds=[0,500]
ionoout=invertRSTO(RSTO,ionoin,alpha_list=alpha_arr,invtype=invtype,rbounds=rbounds)[0]
outfile=os.path.join(basedir,acfloc,'00lags{0}.h5'.format(invtype))
ionoout.saveh5(outfile)
if acfdir=='ACF':
lagsDatasum=ionoout.Param_List
# !!! This is done to speed up development
lagsNoisesum=sp.zeros_like(lagsDatasum)
Nlags=lagsDatasum.shape[-1]
pulses_s=RSTO.simparams['Tint']/RSTO.simparams['IPP']
Ctt=makeCovmat(lagsDatasum,lagsNoisesum,pulses_s,Nlags)
outfile=os.path.join(basedir,acfloc,'00sigs{0}.h5'.format(invtype))
ionoout.Param_List=Ctt
ionoout.Param_Names=sp.repeat(ionoout.Param_Names[:,sp.newaxis],Nlags,axis=1)
ionoout.saveh5(outfile)
def plotdata(ionofile_in,ionofile_fit,madfile,time1):
fig1,axmat =plt.subplots(2,2,facecolor='w',figsize=(10,10))
axvec = axmat.flatten()
paramlist = ['ne','te','ti','vo']
paramlisti = ['Ne','Te','Ti','Vi']
paramlistiname = ['$N_e$','$T_e$','$T_i$','$V_i$']
paramunit = ['$m^{-3}$','$^\circ$ K','$^\circ$ K','m/s']
boundlist = [[0.,7e11],[500.,3200.],[500.,2500.],[-500.,500.]]
IonoF = IonoContainer.readh5(ionofile_fit)
IonoI = IonoContainer.readh5(ionofile_in)
gfit = GeoData(readIono,[IonoF,'spherical'])
ginp = GeoData(readIono,[IonoI,'spherical'])
data1 = GeoData(readMad_hdf5,[madfile,['nel','te','ti','vo','dnel','dte','dti','dvo']])
data1.data['ne']=sp.power(10.,data1.data['nel'])
data1.data['dne']=sp.power(10.,data1.data['dnel'])
t1,t2 = data1.timelisting()[340]
handlist = []
for inum,iax in enumerate(axvec):
ploth = rangevsparam(data1,data1.dataloc[0,1:],time1,gkey=paramlist[inum],fig=fig1,ax=iax,it=False)
handlist.append(ploth[0])
ploth = rangevsparam(ginp,ginp.dataloc[0,1:],0,gkey=paramlisti[inum],fig=fig1,ax=iax,it=False)
handlist.append(ploth[0])
ploth = rangevsparam(gfit,gfit.dataloc[0,1:],0,gkey=paramlisti[inum],fig=fig1,ax=iax,it=False)
handlist.append(ploth[0])
iax.set_xlim(boundlist[inum])
iax.set_ylabel('Altitude in km')
iax.set_xlabel(paramlistiname[inum]+' in '+paramunit[inum])
# with error bars
plt.tight_layout()
fig1.suptitle('Comparison Without Error Bars\nPFISR Data Times: {0} to {1}'.format(t1,t2))
plt.subplots_adjust(top=0.9)
plt.figlegend( handlist[:3], ['PFISR', 'SimISR Input','SimISR Fit'], loc = 'lower center', ncol=5, labelspacing=0. )
fig2,axmat2 =plt.subplots(2,2,facecolor='w',figsize=(10,10))
axvec2 = axmat2.flatten()
handlist2 = []
for inum,iax in enumerate(axvec2):
ploth = rangevsparam(data1,data1.dataloc[0,1:],time1,gkey=paramlist[inum],gkeyerr='d'+paramlist[inum],fig=fig2,ax=iax,it=False)
handlist2.append(ploth[0])
ploth = rangevsparam(ginp,ginp.dataloc[0,1:],0,gkey=paramlisti[inum],fig=fig2,ax=iax,it=False)
handlist2.append(ploth[0])
ploth = rangevsparam(gfit,gfit.dataloc[0,1:],0,gkey=paramlisti[inum],gkeyerr='n'+paramlisti[inum],fig=fig2,ax=iax,it=False)
handlist2.append(ploth[0])
iax.set_xlim(boundlist[inum])
iax.set_ylabel('Altitude in km')
iax.set_xlabel(paramlistiname[inum]+' in '+paramunit[inum])
plt.tight_layout()
fig2.suptitle('Comparison With Error Bars\nPFISR Data Times: {0} to {1}'.format(t1,t2))
plt.subplots_adjust(top=0.9)
plt.figlegend( handlist2[:3], ['PFISR', 'SimISR Input','SimISR Fit'], loc = 'lower center', ncol=5, labelspacing=0. )
return (fig1,axvec,handlist,fig2,axvec2,handlist2)
def startvalfunc(Ne_init, loc, locsp, time, inputs, ambinfo=[0]):
""" This is a method to determine the start values for the fitter.
Inputs
Ne_init - A nloc x nt numpy array of the initial estimate of electron density. Basically
the zeroth lag of the ACF.
loc - A nloc x 3 numpy array of cartisian coordinates.
time - A nt x 2 numpy array of times in seconds
exinputs - A list of extra inputs allowed for by the fitter class. It only
has one element and its the name of the ionocontainer file holding the
rest of the start parameters.
Outputs
xarray - This is a numpy array of starting values for the fitter parameters."""
if isinstance(inputs, str):
if os.path.splitext(inputs)[-1] == '.h5':
ionoin = IonoContainer.readh5(inputs)
elif os.path.splitext(inputs)[-1] == '.mat':
ionoin = IonoContainer.readmat(inputs)
elif os.path.splitext(inputs)[-1] == '':
ionoin = makeionocombined(inputs)
elif isinstance(inputs, list):
ionoin = makeionocombined(inputs)
else:
ionoin = inputs
numel = sp.prod(ionoin.Param_List.shape[-2:]) + 1
xarray = sp.zeros((loc.shape[0], len(time), numel))
for ilocn, iloc in enumerate(loc):
#for iamb in ambinfo:
newlocsp = locsp[ilocn]
#newlocsp[0] += iamb
(datast, vel) = ionoin.getclosestsphere(newlocsp, time)[:2]
datast[:, -1, 0] = Ne_init[ilocn, :]
# get the ion densities
ionoden = datast[:, :-1, 0]
# find the right normalization for the ion species
ionodensum = sp.repeat(sp.sum(ionoden, axis=-1)[:, sp.newaxis],
ionoden.shape[-1],
axis=-1)
# renormalized to the right level
ionoden = sp.repeat(Ne_init[ilocn, :, sp.newaxis],
ionoden.shape[-1],
axis=-1) * ionoden / ionodensum
datast[:, :-1, 0] = ionoden
xarray[ilocn, :, :-1] = sp.reshape(
datast, (len(time), numel - 1)) #/float(len(ambinfo))
locmag = sp.sqrt(sp.sum(iloc * iloc))
ilocmat = sp.repeat(iloc[sp.newaxis, :], len(time), axis=0)
xarray[ilocn, :,
-1] = sp.sum(vel * ilocmat) / locmag #/float(len(ambinfo))
return xarray
def plotspectra(simfile,realfile,coords = [0.,0.,250.],timesim=0,timereal=345):
fmax=50e3
simiono=IonoContainer.readh5(simfile)
realiono=IonoContainer.readh5(realfile)
t1,t2 = realiono.timelisting()[timereal]
fig1 =plt.figure(facecolor='w',figsize=(10,10))
ax0=plt.subplot2grid((2,2),(0,0))
ax1=plt.subplot2grid((2,2),(0,1))
ax2=plt.subplot2grid((2,2),(1,0),colspan=2)
reald_closest=realiono.getclosest(coords)
real_lags=reald_closest[0][timereal]
sim_closest=simiono.getclosest(coords)
sim_lags=sim_closest[0][timesim]
Nlags = sp.minimum(len(sim_lags),len(real_lags))
l =sp.arange(Nlags)
Nspec=128
simspec = scfft.fftshift(scfft.fft(sim_lags[:Nlags],n=128)).real
realspec= scfft.fftshift(scfft.fft(real_lags[:Nlags],n=128)).real
f=1e-3*sp.arange(-sp.floor(float(Nspec)/2.),sp.ceil(float(Nspec)/2.)) * fmax/Nspec
handlist = []
handlist.append(ax0.plot(l,real_lags[:Nlags].real,'b-',label='Real')[0])
handlist.append(ax0.plot(l,sim_lags[:Nlags].real,'r-',label='Sim')[0])
ax0.set_title('Real Part of ACF')
ax0.set_xlabel('Lag Number')
ax0.set_ylabel('Amplitude')
handlist.append(ax1.plot(l,real_lags[:Nlags].imag,'b-',label='Real')[0])
handlist.append(ax1.plot(l,sim_lags[:Nlags].imag,'r-',label='Sim')[0])
ax1.set_title('Imaginary Part of ACF')
ax1.set_xlabel('Lag Number')
handlist.append(ax2.plot(f,realspec,'b-',label='Real')[0])
handlist.append(ax2.plot(f,simspec,'r-',label='Sim')[0])
ax2.set_title('Spectra')
ax2.set_xlabel('Frequency in kHz')
ax2.set_ylabel('Amplitude')
plt.figlegend( handlist[:2], ['PFISR', 'SimISR'], loc = 'lower center', ncol=5, labelspacing=0. )
fig1.suptitle('Location: [{2} km,{3} km, {4} km] \nPFISR Data Times: {0} to {1}'.format(t1,t2,*coords))
plt.subplots_adjust(top=0.9)
return fig1,handlist
def makedata(testpath):
"""
This will make the input data for the test case. The data will have the
default set of parameters Ne=Ne=1e11 and Te=Ti=2000.
Inputs
testpath - Directory that will hold the data.
"""
finalpath = testpath.joinpath('Origparams')
if not finalpath.exists():
finalpath.mkdir()
data = SIMVALUES
z = sp.linspace(50., 1e3, 50)
nz = len(z)
params = sp.tile(data[sp.newaxis, sp.newaxis, :, :], (nz, 1, 1, 1))
coords = sp.column_stack((sp.ones(nz), sp.ones(nz), z))
species = ['O+', 'e-']
times = sp.array([[0, 1e9]])
vel = sp.zeros((nz, 1, 3))
Icont1 = IonoContainer(coordlist=coords,
paramlist=params,
times=times,
sensor_loc=sp.zeros(3),
ver=0,
coordvecs=['x', 'y', 'z'],
paramnames=None,
species=species,
velocity=vel)
finalfile = finalpath.joinpath('0 stats.h5')
Icont1.saveh5(str(finalfile))
# set start temp to 1000 K.
Icont1.Param_List[:, :, :, 1] = 1e3
Icont1.saveh5(str(testpath.joinpath('startfile.h5')))
def plotbackground(testdir,figname):
"""
Plots the background densities and temperatures.
"""
sns.set_style('whitegrid')
sns.set_context('notebook')
sns.color_palette("hls", 8)
filelist = glob.glob(os.path.join(testdir,'Origparams','*.h5'))
numlist = [os.path.splitext(os.path.split(x)[-1])[0] for x in filelist]
numdict = {numlist[i]:filelist[i] for i in range(len(filelist))}
ylim = [100.,500.]
slist = sorted(numlist,key=ke)
ifile = numdict[slist[0]]
Iono_in = IonoContainer.readh5(ifile)
numz=sp.sum(Iono_in.Cart_Coords[:,2]==Iono_in.Cart_Coords[0,2])
redcoords=Iono_in.Cart_Coords[numz/2::numz]
z=redcoords[:,2]
begdata = Iono_in.Param_List[numz/2::numz]
allni=begdata[:,0,:-1,0]
allti=begdata[:,0,:-1,1]
T_all = begdata[:,0,:,1]
N_all=begdata[:,0,:,0]
Te = begdata[:,0,-1,-1]
Ne = begdata[:,0,-1,0]
tini=allti*allni
Nisum=allni.sum(-1)
Ti=tini.sum(-1)/Nisum
ncols=2
figsize = (5*ncols,7)
handlist=[]
fig ,axmat= plt.subplots(nrows=1,ncols=ncols,facecolor='w',figsize=figsize,sharey=True)
for i in range(len(Iono_in.Species)):
p1=axmat[0].plot(N_all[:,i],z,label=Iono_in.Species[i],linewidth=lw)[0]
handlist.append(p1)
axmat[0].set_ylabel('Alt in km',fontsize=fs)
axmat[0].set_ylim(ylim)
axmat[0].set_xlabel(r'Number Density in m$^{-3}$',fontsize=fs)
axmat[0].set_title('Ion and Electron Densities',fontsize=fs)
axmat[0].set_xscale('log')
axmat[0].set_xlim([1e2,3e11])
axmat[0].legend(handlist,Iono_in.Species,loc='upper left',fontsize='large')
# handlist2=[]
p2=axmat[1].plot(Te,z,Ti,z,linewidth=lw)
# for i in range(len(Iono_in.Species)):
# p1=axmat[1].plot(T_all[:,i],z,label=Iono_in.Species[i],linewidth=lw)[0]
# handlist2.append(p1)
axmat[1].set_title('Ion and Electron Temperatures',fontsize=fs)
axmat[1].set_xlabel(r'Temperature in $^{\circ}$K',fontsize=fs)
axmat[1].set_xlim([0,2e3])
axmat[1].set_ylim(ylim)
# axmat[1].legend(handlist2,Iono_in.Species,loc='upper left',fontsize='large')
axmat[1].legend(p2,['Te','Ti'],loc='upper left')
fig.savefig(figname,dpi=300)
def makedata(testpath, tint):
""" This will make the input data for the test case. The data will have cases
where there will be enhancements in Ne, Ti and Te in one location. Each
case will have 3 integration periods. The first 3 integration periods will
be the default set of parameters Ne=Ne=1e11 and Te=Ti=2000.
Inputs
testpath - Directory that will hold the data.
tint - The integration time in seconds.
"""
testpath = Path(testpath).expanduser()
finalpath = testpath.joinpath('Origparams')
if not finalpath.is_dir():
finalpath.mkdir()
data = sp.array([[1e11, 1100.], [1e11, 2100.]])
z = (50. + sp.arange(50) * 10.)
nz = len(z)
params = sp.tile(data[sp.newaxis, sp.newaxis], (nz, 1, 1, 1))
epnt = range(20, 22)
p2 = sp.tile(params, (1, 4, 1, 1))
#enhancement in Ne
p2[epnt, 1, :, 0] = 5e11
#enhancement in Ti
p2[epnt, 2, 0, 1] = 2200.
#enhancement in Te
p2[epnt, 3, 1, 1] = 4200.
coords = sp.column_stack((sp.ones(nz), sp.ones(nz), z))
species = ['O+', 'e-']
times = sp.array([[0, 1e3]])
times2 = sp.column_stack((sp.arange(0, 4), sp.arange(1, 5))) * 3 * tint
vel = sp.zeros((nz, 1, 3))
vel2 = sp.zeros((nz, 4, 3))
Icontstart = IonoContainer(coordlist=coords,
paramlist=params,
times=times,
sensor_loc=sp.zeros(3),
ver=0,
coordvecs=['x', 'y', 'z'],
paramnames=None,
species=species,
velocity=vel)
Icont1 = IonoContainer(coordlist=coords,
paramlist=p2,
times=times2,
sensor_loc=sp.zeros(3),
ver=0,
coordvecs=['x', 'y', 'z'],
paramnames=None,
species=species,
velocity=vel2)
finalfile = finalpath.joinpath('0 stats.h5')
Icont1.saveh5(str(finalfile))
Icontstart.saveh5(str(testpath.joinpath('startfile.h5')))
def fit2geodata(inputfile):
Ionoin = IonoContainer.readh5(inputfile)
keep = ["Ne", "Te", "Ti"]
GD = GeoData(readIono, [Ionoin])
for ikey in GD.data.keys():
if ikey not in keep:
del GD.data[ikey]
indir, fnameall = os.path.split(inputfile)
fname = os.path.splitext(fnameall)[0]
GD.write_h5(os.path.join(indir, fname + "GEOD.h5"))
print("Saved " + os.path.join(indir, fname + ".h5"))
def makedata(testpath):
""" This will make the input data for the test case. The data will have the
default set of parameters Ne=Ne=1e11 and Te=Ti=2000.
Inputs
testpath - Directory that will hold the data.
"""
finalpath = testpath.joinpath('Origparams')
if not finalpath.exists():
finalpath.mkdir()
data=SIMVALUES
z = sp.linspace(50.,1e3,50)
nz = len(z)
params = sp.tile(data[sp.newaxis,sp.newaxis,:,:],(nz,1,1,1))
coords = sp.column_stack((sp.ones(nz),sp.ones(nz),z))
species=['O+','e-']
times = sp.array([[0,1e3]])
vel = sp.zeros((nz,1,3))
Icont1 = IonoContainer(coordlist=coords,paramlist=params,times = times,sensor_loc = sp.zeros(3),ver =0,coordvecs =
['x','y','z'],paramnames=None,species=species,velocity=vel)
finalfile = finalpath.joinpath('0 stats.h5')
Icont1.saveh5(str(finalfile))
# set start temp to 1000 K.
Icont1.Param_List[:,:,:,1]=1e3
Icont1.saveh5(str(testpath.joinpath('startfile.h5')))
def fixspecs(basedirlist):
for ibase in basedirlist:
filelist = glob.glob(os.path.join(ibase,'Origparams','*.h5'))
numlist = [os.path.splitext(os.path.split(x)[-1])[0] for x in filelist]
numdict = {numlist[i]:filelist[i] for i in range(len(filelist))}
slist = sorted(numlist,key=ke)
origlist = [numdict[i] for i in slist]
filelist = glob.glob(os.path.join(ibase,'Spectrums','*.h5'))
numlist = [os.path.splitext(os.path.split(x)[-1])[0] for x in filelist]
numdict = {numlist[i]:filelist[i] for i in range(len(filelist))}
slist = sorted(numlist,key=ke)
speclist = [numdict[i] for i in slist]
for (iorig,ispec) in zip(origlist,speclist):
origiono=IonoContainer.readh5(iorig)
speciono=IonoContainer.readh5(ispec)
speciono.Cart_Coords=origiono.Cart_Coords
speciono.Sphere_Coords=origiono.Sphere_Coords
os.remove(ispec)
speciono.saveh5(ispec)
def run_sim(args_commd):
fitoutput = fitterone.fitdata(specfuncs.ISRSfitfunction,
fitterone.simparams['startfile'], fittimes=fitlist,
printlines=printlines)
(fitteddata, fittederror, funcevals, fittedcov) = fitoutput
fittederronly = sp.sqrt(fittederror)
paramnames = []
species = fitterone.simparams['species']
# Seperate Ti and put it in as an element of the ionocontainer.
Ti = fitteddata[:, :, 1]
nTi = fittederronly[:, :, 1]
nTiTe = fittedcov[:, :, 0, 1]
nTiNe = fittedcov[:, :, 0, 2]
nTiVi = fittedcov[:, :, 0, 3]
nTeNe = fittedcov[:, :, 1, 2]
nTeVi = fittedcov[:, :, 1, 3]
nNeVi = fittedcov[:, :, 2, 3]
cov_list = [nTiTe[:, :, sp.newaxis], nTiNe[:, :, sp.newaxis],
nTiVi[:, :, sp.newaxis], nTeNe[:, :, sp.newaxis],
nTeVi[:, :, sp.newaxis], nNeVi[:, :, sp.newaxis]]
cov_list_names = ['nTiTe', 'nTiNe', 'nTiVi', 'nTeNe', 'nTeVi','nNeVi']
paramlist = sp.concatenate([fitteddata, Ti[:, :, sp.newaxis], fittederronly,
nTi[:, :, sp.newaxis], funcevals[:, :, sp.newaxis]]
+ cov_list, axis=2)
ionoout = IonoContainer(Ionoin.Cart_Coords, paramlist.real, timevec, ver=newver,
coordvecs=Ionoin.Coord_Vecs, paramnames=paranamsf,
species=species)
ionoout.saveh5(str(outfile))
def makespectrums(basedir, configfile, printlines=True):
""" This will make all of the spectra for a set of data and save it in a
folder in basedir called Spectrums. It is assumed that the data in the Origparams
is time tagged in the title with a string seperated by a white space and the
rest of the file name. For example ~/DATA/Basedir/0 origdata.h5.
Inputs:
basedir: A string for the directory that will hold all of the data for the simulation.
configfile: The configuration file for the simulation.
"""
basedir = Path(basedir).expanduser()
dirio = ('Origparams', 'Spectrums')
inputdir = basedir/dirio[0]
outputdir = basedir/dirio[1]
# determine the list of h5 files in Origparams directory
dirlist = sorted(inputdir.glob('*.h5'))
# Make the lists of numbers and file names for the dictionary
(listorder, _, _, timebeg, _) = IonoContainer.gettimes(dirlist)
slist = [dirlist[ikey] for ikey in listorder]
(sensdict, simparams) = readconfigfile(configfile)
# Delete data
outfiles = outputdir.glob('*.h5')
for ifile in outfiles:
ifile.unlink()
for inum, curfile in zip(timebeg, slist):
outfile = outputdir / (str(inum)+' spectrum.h5')
update_progress(float(inum)/float(len(slist)),
'Processing file {} starting at {}'.format(curfile.name, datetime.now()))
curiono = IonoContainer.readh5(str(curfile))
curiono.makespectruminstanceopen(specfuncs.ISRSspecmake, sensdict,
int(simparams['numpoints']), float(inum),
float(len(slist)), printlines).saveh5(str(outfile))
update_progress(float(inum+1)/float(len(slist)),
'Finished file {} starting at {}'.format(curfile.name, datetime.now()))
def pyglowinput(
latlonalt=[65.1367, -147.4472, 250.00],
dn_list=[datetime(2015, 3, 21, 8, 00),
datetime(2015, 3, 21, 20, 00)],
z=None):
if z is None:
z = sp.linspace(50., 1000., 200)
dn_diff = sp.diff(dn_list)
dn_diff_sec = dn_diff[-1].seconds
timelist = sp.array([calendar.timegm(i.timetuple()) for i in dn_list])
time_arr = sp.column_stack((timelist, sp.roll(timelist, -1)))
time_arr[-1, -1] = time_arr[-1, 0] + dn_diff_sec
v = []
coords = sp.column_stack((sp.zeros((len(z), 2), dtype=z.dtype), z))
all_spec = ['O+', 'NO+', 'O2+', 'H+', 'HE+']
Param_List = sp.zeros((len(z), len(dn_list), len(all_spec), 2))
for idn, dn in enumerate(dn_list):
for iz, zcur in enumerate(z):
latlonalt[2] = zcur
pt = Point(dn, *latlonalt)
pt.run_igrf()
pt.run_msis()
pt.run_iri()
# so the zonal pt.u and meriodinal winds pt.v will coorispond to x and y even though they are
# supposed to be east west and north south. Pyglow does not seem to have
# vertical winds.
v.append([pt.u, pt.v, 0])
for is1, ispec in enumerate(all_spec):
Param_List[iz, idn, is1, 0] = pt.ni[ispec] * 1e6
Param_List[iz, idn, :, 1] = pt.Ti
Param_List[iz, idn, -1, 0] = pt.ne * 1e6
Param_List[iz, idn, -1, 1] = pt.Te
Param_sum = Param_List[:, :, :, 0].sum(0).sum(0)
spec_keep = Param_sum > 0.
species = sp.array(all_spec)[spec_keep[:-1]].tolist()
species.append('e-')
Param_List[:, :] = Param_List[:, :, spec_keep]
Iono_out = IonoContainer(coords,
Param_List,
times=time_arr,
species=species)
return Iono_out
def plotparams(datapath,indch):
# read in the files
geofile = os.path.join(datapath,'Fitted','fitteddataGEOD.h5')
acffile = os.path.join(datapath,'ACF','00lags.h5')
geod = GeoData.read_h5(geofile)
acfIono = IonoContainer.readh5(acffile)
picklefile = os.path.join(datapath,'config.ini')
(sensdict,simparams) = readconfigfile(picklefile)
#determine the ind
dataloc = geod.dataloc
rngloc = 210
ind = sp.argmin(sp.absolute(dataloc[:,0]-rngloc))
#
Ne = geod.data['Ne'][ind]
Ti = geod.data['Ti'][ind]
Te = geod.data['Te'][ind]
#make a spectrum
npts = 128
datablock = sp.array([[Ne[indch],Ti[indch]],[Ne[indch],Te[indch]]])
specobj = ISRSpectrum(centerFrequency =sensdict['fc'],nspec = npts,sampfreq=sensdict['fs'])
(omeg,specexample,rcs) = specobj.getspecsep(datablock,simparams['species'],rcsflag=True)
specexample = rcs*npts*specexample/sp.absolute(specexample).sum()
acf = acfIono.Param_List[ind,indch]
origspec = scfft.fftshift(scfft.fft(acf,n=npts)).real
(figmplf, [[ax1,ax2],[ax3,ax4]]) = plt.subplots(2, 2,figsize=(16, 12), facecolor='w')
ax1.plot(sp.arange(len(Ne)),Ne)
ax1.set_title('$N_e$')
ax2.plot(sp.arange(len(Ti)),Ti)
ax2.set_title('$T_i$')
ax3.plot(sp.arange(len(Te)),Te)
ax3.set_title('$T_e$');
spec1 = ax4.plot(omeg*1e-3,origspec,label='Measured')
spec2 = ax4.plot(omeg*1e-3,specexample,label='Fitted')
ax4.set_title('Measured and Fitted Spectrums')
ax4.set_xlabel('Frequency KHz')
ax4.legend(loc = 1)
# Nevals = sp.linspace(5e10,5e11,10)
# Tivals = sp.linspace(1e3,2e5,10)
# Tevals = sp.linspace(1e3,2e5,10)
# xlist = [[1],Tivals,Nevals,Tevals,[0]]
# outsurf = makefitsurf(xlist,acf,sensdict,simparams)
return(figmplf)
def applymat(basedir, configfile, optinputs):
"""
This function apply the matrix version of the space time ambiugty function
to the ACFs and save the outcome in h5 files within the directory ACFMat.
Inputs:
basedir: A string for the directory that will hold all of the data for the simulation.
configfile: The configuration file for the simulation.
"""
dirio = ('Spectrums', 'Mat', 'ACFMat')
basedir = Path(basedir)
inputdir = basedir.joinpath(dirio[0])
outputdir2 = basedir.joinpath(dirio[2])
dirlist = [str(i) for i in inputdir.glob('*.h5')]
(listorder, timevector, _, _, _) = IonoContainer.gettimes(dirlist)
ionolist = [dirlist[ikey] for ikey in listorder]
rsto = RadarSpaceTimeOperator(ionolist, configfile, timevector, mattype='matrix')
ionoout = rsto.mult_iono(ionolist)
outfile = outputdir2.joinpath('00lags.h5')
ionoout.saveh5(str(outfile))
def makeradardata(basedir,configfile,remakealldata):
""" This function will make the radar data and create the acf estimates.
Inputs:
basedir: A string for the directory that will hold all of the data for the simulation.
configfile: The configuration file for the simulation.
remakealldata: A bool that determines if the radar data is remade. If false
only the acfs will be estimated using the radar that is already made."""
dirio = ('Spectrums', 'Radardata', 'ACF')
inputdir = basedir/dirio[0]
outputdir = basedir/dirio[1]
outputdir2 = basedir/dirio[2]
# determine the list of h5 files in Origparams directory
dirlist = [str(i) for i in inputdir.glob('*.h5')]
# Make the lists of numbers and file names for the dictionary
if len(dirlist) > 0:
(listorder, _, _, timebeg, _) = IonoContainer.gettimes(dirlist)
Ionodict = {timebeg[itn]:dirlist[it] for itn, it in enumerate(listorder)}
else:
Ionodict = {0.:str(inputdir.joinpath('00.h5'))}
# Find all of the raw data files
radardatalist = outputdir.glob('*RawData.h5')
if radardatalist and (not remakealldata):
# XXX need to work on time stuff
outlist2 = radardatalist
else:
outlist2 = None
# create the radar data file class
rdata = RadarDataFile(Ionodict, configfile, outputdir, outfilelist=outlist2)
# From the ACFs and uncertainties
(ionoout, ionosig) = rdata.processdataiono()
# save the acfs and uncertianties in ionocontainer h5 files.
ionoout.saveh5(str(outputdir2.joinpath('00lags.h5')))
ionosig.saveh5(str(outputdir2.joinpath('00sigs.h5')))
return ()
def makedata(testpath,tint):
""" This will make the input data for the test case. The data will have cases
where there will be enhancements in Ne, Ti and Te in one location. Each
case will have 3 integration periods. The first 3 integration periods will
be the default set of parameters Ne=Ne=1e11 and Te=Ti=2000.
Inputs
testpath - Directory that will hold the data.
tint - The integration time in seconds.
"""
testpath = Path(testpath).expanduser()
finalpath = testpath.joinpath('Origparams')
if not finalpath.is_dir():
finalpath.mkdir()
data = sp.array([[1e11,1100.], [1e11,2100.]])
z = (50.+sp.arange(50)*10.)
nz = len(z)
params = sp.tile(data[sp.newaxis, sp.newaxis],(nz, 1, 1, 1))
epnt = range(20,22)
p2 = sp.tile(params, (1, 4, 1, 1))
#enhancement in Ne
p2[epnt, 1, :, 0] = 5e11
#enhancement in Ti
p2[epnt,2,0,1] = 2200.
#enhancement in Te
p2[epnt,3,1,1] = 4200.
coords = sp.column_stack((sp.zeros(nz), sp.zeros(nz), z))
species=['O+', 'e-']
times = sp.array([[0, 1e3]])
times2 = sp.column_stack((sp.arange(0, 4), sp.arange(1, 5)))*3*tint
vel = sp.zeros((nz, 1, 3))
vel2 = sp.zeros((nz, 4, 3))
Icontstart = IonoContainer(coordlist=coords, paramlist=params, times=times,
sensor_loc=sp.zeros(3), ver=0, coordvecs=['x', 'y', 'z'],
paramnames=None, species=species, velocity=vel)
Icont1 = IonoContainer(coordlist=coords, paramlist=p2, times=times2,
sensor_loc=sp.zeros(3), ver=0, coordvecs=['x', 'y', 'z'],
paramnames=None, species=species, velocity=vel2)
finalfile = finalpath.joinpath('0 stats.h5')
Icont1.saveh5(str(finalfile))
Icontstart.saveh5(str(testpath.joinpath('startfile.h5')))
def makedata(testpath,tint):
""" This will make the input data for the test case. The data will have cases
where there will be enhancements in Ne, Ti and Te in one location. Each
case will have 3 integration periods. The first 3 integration periods will
be the default set of parameters Ne=Ne=1e11 and Te=Ti=2000.
Inputs
testpath - Directory that will hold the data.
tint - The integration time in seconds.
"""
finalpath = os.path.join(testpath,'Origparams')
if not os.path.isdir(finalpath):
os.mkdir(finalpath)
z = sp.linspace(50.,750,150)
nz = len(z)
Z_0 = 250.
H_0=30.
N_0=6.5e11
c1 = Chapmanfunc(z,H_0,Z_0,N_0)+5e10
z0=50.
T0=600.
Te,Ti = TempProfile(z,T0,z0)
params = sp.zeros((nz,1,2,2))
params[:,0,0,0] = c1
params[:,0,1,0] = c1
params[:,0,0,1] = Ti
params[:,0,1,1] = Te
coords = sp.column_stack((sp.zeros(nz),sp.zeros(nz),z))
species=['O+','e-']
times = sp.array([[0,1e3]])
times2 = sp.column_stack((sp.arange(0,1),sp.arange(1,2)))*3*tint
vel = sp.zeros((nz,1,3))
vel2 = sp.zeros((nz,4,3))
Icontstart = IonoContainer(coordlist=coords,paramlist=params,times = times,sensor_loc = sp.zeros(3),ver =0,coordvecs =
['x','y','z'],paramnames=None,species=species,velocity=vel)
Icont1 = IonoContainer(coordlist=coords,paramlist=params,times = times,sensor_loc = sp.zeros(3),ver =0,coordvecs =
['x','y','z'],paramnames=None,species=species,velocity=vel2)
finalfile = os.path.join(finalpath,'0 stats.h5')
Icont1.saveh5(finalfile)
Icontstart.saveh5(os.path.join(testpath,'startfile.h5'))
def runfitter(paramdict, SNR, n_pulses, n_runs, Niratio, x_0=SVALS):
"""
paramdict
"""
data = SIMVALUES
z = sp.linspace(50., 1e3, 50)
nz = len(z)
params = sp.tile(data[sp.newaxis, sp.newaxis, :, :], (nz, 1, 1, 1))
coords = sp.column_stack((sp.ones(nz), sp.ones(nz), z))
species = ['O+', 'e-']
times = sp.array([[0, 1e9]])
vel = sp.zeros((nz, 1, 3))
(sensdict, simparams) = readconfigfile(defcon)
species = paramdict['species']
nspecies = len(species)
ni = nspecies-1
Icont1 = IonoContainer(coordlist=coords, paramlist=params, times=times,
sensor_loc=sp.zeros(3), ver=0, coordvecs=['x', 'y', 'z'],
paramnames=None, species=species, velocity=vel)
omeg, outspecs = specfuncs.ISRSspecmake(Icont1, sensdict, int(simparams['numpoints']))
tau, acf = spect2acf(omeg, outspecs)
t_log = sp.logical_and(tau<simparams['Pulselength'],tau>=0)
plen = t_log.sum()
amb_dict = simparams['amb_dict']
amb_mat_flat = sp.zeros_like(amb_dict['WttMatrix'])
eyemat = sp.eye(plen)
amb_mat_flat[:plen, t_log] = eyemat
fitmode = simparams['fitmode']
simparams['amb_dict']['WttMatrix'] = amb_mat_flat
Nloc, Nt = acf.shape[:2]
# output Files
fittedarray = sp.zeros((Nloc, Nt, nparams+1))*sp.nan
fittederror = sp.zeros((Nloc, Nt, nparams+1))*sp.nan
fittedcov = sp.zeros((Nloc, Nt, 4, 4))*sp.nan
funcevals = sp.zeros((Nloc, Nt))
for iloc, ilocvec in acf:
for itn, iacf in ilocvec:
curlag = sp.dot(amb_mat_flat, iacf)
std_lag = sp.absolute(curlag)(1.+1./SNR)/sp.sqrt(n_pulses)
covar_lag = sp.diag(sp.power(std_lag,2.))
curlag = curlag + std_lag*sp.random.randn(curlag.shape)
d_func = (curlag, sensdict, simparams, Niratio)
# Only fit Ti, Te, Ne and Vi
# change variables because of new fit mode
# Perform the fitting
optresults = scipy.optimize.least_squares(fun=specfuncs.ISRSfitfunction, x0=x_0,
method='lm', verbose=0, args=d_func)
x_res = optresults.x.real
# Derive data for the ions using output from the fitter and ion
# species ratios which are assumed to be given.
ionstuff = sp.zeros(ni*2-1)
ionstuff[:2*ni:2] = x_res[1]*Niratio
ionstuff[1:2*ni-1:2] = x_res[0]
# change variables because of new fit mode
if fitmode == 1:
x_res[2] = x_res[2]*x_res[0]
fittedarray[iloc, itn] = sp.append(ionstuff,
sp.append(x_res, Ne_start[iloc, itime]))
funcevals[iloc, itn] = optresults.nfev
# fittedarray[iloc,itime] = sp.append(optresults.x,Ne_start[iloc,itime])
resid = optresults.cost
jac = optresults.jac
# combine the rows because of the complex conjugates
jacc = jac[0::2]+jac[1::2]
try:
# Derive covariances for the ions using output from the fitter and ion species ratios which are assumed to be given.
#covf = sp.linalg.inv(sp.dot(jac.transpose(),jac))*resid/dof
covf = sp.linalg.inv(sp.dot(sp.dot(jacc.transpose(),
sp.linalg.inv(covar_lag)), jacc))
# change variables because of new fit mode
if fitmode == 1:
# is this right?
covf[2] = covf[2]*x_res[0]
covf[:,2] = covf[:,2]*x_res[0]
vars_vec = sp.diag(covf).real
ionstuff = sp.zeros(ni*2-1)
ionstuff[:2*ni:2] = vars_vec[1]*Niratio
ionstuff[1:2*ni-1:2] = vars_vec[0]
vars_vec = sp.append(ionstuff, vars_vec)
fittedcov[iloc, itn] = covf
except:#sp.linalg.LinAlgError('singular matrix'):
vars_vec = sp.ones(nparams)*float('nan')
# if len(vars_vec)<fittederror.shape[-1]-1:
# pdb.set_trace()
fittederror[iloc, itn, :-1] = vars_vec
return(fittedarray, fittederror, funcevals, fittedcov)
def __init__(self, Ionodict, inifile, outdir, outfilelist=None):
"""
This function will create an instance of the RadarData class. It will
take in the values and create the class and make raw IQ data.
Inputs:
sensdict - A dictionary of sensor parameters
angles - A list of tuples which the first position is the az angle
and the second position is the el angle.
IPP - The interpulse period in seconds represented as a float.
Tint - The integration time in seconds as a float. This will be the
integration time of all of the beams.
time_lim - The length of time of the simulation the number of time points
will be calculated.
pulse - A numpy array that represents the pulse shape.
rng_lims - A numpy array of length 2 that holds the min and max range
that the radar will cover.
"""
(sensdict, simparams) = readconfigfile(inifile)
self.simparams = simparams
N_angles = len(self.simparams['angles'])
NNs = int(self.simparams['NNs'])
self.sensdict = sensdict
Npall = sp.floor(self.simparams['TimeLim']/self.simparams['IPP'])
Npall = int(sp.floor(Npall/N_angles)*N_angles)
Np = Npall/N_angles
print("All spectrums created already")
filetimes = Ionodict.keys()
filetimes.sort()
ftimes = sp.array(filetimes)
simdtype = self.simparams['dtype']
pulsetimes = sp.arange(Npall)*self.simparams['IPP'] +ftimes.min()
pulsefile = sp.array([sp.where(itimes-ftimes >= 0)[0][-1] for itimes in pulsetimes])
# differentiate between phased arrays and dish antennas
if sensdict['Name'].lower() in ['risr', 'pfisr', 'risr-n']:
beams = sp.tile(sp.arange(N_angles), Npall/N_angles)
else:
# for dish arrays
brate = simparams['beamrate']
beams2 = sp.repeat(sp.arange(N_angles), brate)
beam3 = sp.concatenate((beams2, beams2[::-1]))
ntile = int(sp.ceil(Npall/len(beam3)))
leftover = int(Npall-ntile*len(beam3))
if ntile > 0:
beams = sp.tile(beam3, ntile)
beams = sp.concatenate((beams, beam3[:leftover]))
else:
beams = beam3[:leftover]
pulsen = sp.repeat(sp.arange(Np), N_angles)
pt_list = []
pb_list = []
pn_list = []
fname_list = []
self.datadir = outdir
self.maindir = outdir.parent
self.procdir = self.maindir/'ACF'
Nf = len(filetimes)
progstr = 'Data from {:d} of {:d} being processed Name: {:s}.'
if outfilelist is None:
print('\nData Now being created.')
Noisepwr = v_Boltz*sensdict['Tsys']*sensdict['BandWidth']
self.outfilelist = []
for ifn, ifilet in enumerate(filetimes):
outdict = {}
ifile = Ionodict[ifilet]
ifilename = Path(ifile).name
update_progress(float(ifn)/Nf, progstr.format(ifn, Nf, ifilename))
curcontainer = IonoContainer.readh5(ifile)
if ifn == 0:
self.timeoffset = curcontainer.Time_Vector[0, 0]
pnts = pulsefile == ifn
pt = pulsetimes[pnts]
pb = beams[pnts]
pn = pulsen[pnts].astype(int)
rawdata = self.__makeTime__(pt, curcontainer.Time_Vector,
curcontainer.Sphere_Coords,
curcontainer.Param_List, pb)
d_shape = rawdata.shape
n_tempr = sp.random.randn(*d_shape).astype(simdtype)
n_tempi = 1j*sp.random.randn(*d_shape).astype(simdtype)
noise = sp.sqrt(Noisepwr/2)*(n_tempr+n_tempi)
outdict['AddedNoise'] = noise
outdict['RawData'] = rawdata+noise
outdict['RawDatanonoise'] = rawdata
outdict['NoiseData'] = sp.sqrt(Noisepwr/2)*(sp.random.randn(len(pn), NNs).astype(simdtype)+
1j*sp.random.randn(len(pn), NNs).astype(simdtype))
outdict['Pulses'] = pn
outdict['Beams'] = pb
outdict['Time'] = pt
fname = '{0:d} RawData.h5'.format(ifn)
newfn = self.datadir/fname
self.outfilelist.append(str(newfn))
dict2h5(str(newfn), outdict)
#Listing info
pt_list.append(pt)
pb_list.append(pb)
pn_list.append(pn)
fname_list.append(fname)
infodict = {'Files':fname_list, 'Time':pt_list, 'Beams':pb_list, 'Pulses':pn_list}
dict2h5(str(outdir.joinpath('INFO.h5')), infodict)
else:
infodict = h52dict(str(outdir.joinpath('INFO.h5')))
alltime = sp.hstack(infodict['Time'])
self.timeoffset = alltime.min()
self.outfilelist = outfilelist
def makehistdata(params, maindir):
""" This will make the histogram data for the statistics.
Inputs
params - A list of parameters that will have statistics created
maindir - The directory that the simulation data is held.
Outputs
datadict - A dictionary with the data values in numpy arrays. The keys are param names.
errordict - A dictionary with the data values in numpy arrays. The keys are param names.
errdictrel - A dictionary with the error values in numpy arrays, normalized by the correct value. The keys are param names.
"""
maindir = Path(maindir)
ffit = maindir.joinpath('Fitted', 'fitteddata.h5')
inputfiledir = maindir.joinpath('Origparams')
paramslower = [ip.lower() for ip in params]
eparamslower = ['n' + ip.lower() for ip in params]
# set up data dictionary
errordict = {ip: [] for ip in params}
errordictrel = {ip: [] for ip in params}
#Read in fitted data
Ionofit = IonoContainer.readh5(str(ffit))
times = Ionofit.Time_Vector
dataloc = Ionofit.Sphere_Coords
rng = dataloc[:, 0]
rng_log = sp.logical_and(rng > 200., rng < 400)
dataloc_out = dataloc[rng_log]
pnames = Ionofit.Param_Names
pnameslower = sp.array([ip.lower() for ip in pnames.flatten()])
p2fit = [
sp.argwhere(ip == pnameslower)[0][0] if ip in pnameslower else None
for ip in paramslower
]
datadict = {
ip: Ionofit.Param_List[rng_log, :, p2fit[ipn]].flatten()
for ipn, ip in enumerate(params)
}
ep2fit = [
sp.argwhere(ip == pnameslower)[0][0] if ip in pnameslower else None
for ip in eparamslower
]
edatadict = {
ip: Ionofit.Param_List[rng_log, :, ep2fit[ipn]].flatten()
for ipn, ip in enumerate(params)
}
# Determine which input files are to be used.
dirlist = [str(i) for i in inputfiledir.glob('*.h5')]
_, outime, filelisting, _, _ = IonoContainer.gettimes(dirlist)
time2files = []
for itn, itime in enumerate(times):
log1 = (outime[:, 0] >= itime[0]) & (outime[:, 0] < itime[1])
log2 = (outime[:, 1] > itime[0]) & (outime[:, 1] <= itime[1])
log3 = (outime[:, 0] <= itime[0]) & (outime[:, 1] > itime[1])
tempindx = sp.where(log1 | log2 | log3)[0]
time2files.append(filelisting[tempindx])
curfilenum = -1
for iparam, pname in enumerate(params):
curparm = paramslower[iparam]
# Use Ne from input to compare the ne derived from the power.
if curparm == 'nepow':
curparm = 'ne'
datalist = []
for itn, itime in enumerate(times):
for filenum in time2files[itn]:
filenum = int(filenum)
if curfilenum != filenum:
curfilenum = filenum
datafilename = dirlist[filenum]
Ionoin = IonoContainer.readh5(datafilename)
if ('ti' in paramslower) or ('vi' in paramslower):
Ionoin = maketi(Ionoin)
pnames = Ionoin.Param_Names
pnameslowerin = sp.array(
[ip.lower() for ip in pnames.flatten()])
prmloc = sp.argwhere(curparm == pnameslowerin)
if prmloc.size != 0:
curprm = prmloc[0][0]
# build up parameter vector bs the range values by finding the closest point in space in the input
curdata = sp.zeros(len(dataloc_out))
for irngn, curcoord in enumerate(dataloc_out):
tempin = Ionoin.getclosestsphere(curcoord, [itime])[0]
Ntloc = tempin.shape[0]
tempin = sp.reshape(tempin, (Ntloc, len(pnameslowerin)))
curdata[irngn] = tempin[0, curprm]
datalist.append(curdata)
errordict[pname] = datadict[pname] - sp.hstack(datalist)
errordictrel[pname] = 100. * errordict[pname] / sp.absolute(
sp.hstack(datalist))
return datadict, errordict, errordictrel, edatadict
def parametersweep(basedir,configfile,acfdir='ACF',invtype='tik'):
"""
This function will run the inversion numerious times with different constraint
parameters. This will create a directory called cost and place.
Input
basedir - The directory that holds all of the data for the simulator.
configfile - The ini file for the simulation.
acfdir - The directory within basedir that hold the acfs to be inverted.
invtype - The inversion method that will be tested. Can be tik, tikd, and tv.
"""
alpha_sweep=sp.logspace(-3.5,sp.log10(7),25)
costdir = os.path.join(basedir,'Cost')
ionoinfname=os.path.join(basedir,acfdir,'00lags.h5')
ionoin=IonoContainer.readh5(ionoinfname)
dirio = ('Spectrums','Mat','ACFMat')
inputdir = os.path.join(basedir,dirio[0])
dirlist = glob.glob(os.path.join(inputdir,'*.h5'))
(listorder,timevector,filenumbering,timebeg,time_s) = IonoContainer.gettimes(dirlist)
Ionolist = [dirlist[ikey] for ikey in listorder]
RSTO = RadarSpaceTimeOperator(Ionolist,configfile,timevector,mattype='Sim')
npts=RSTO.simparams['numpoints']
ionospec=makeionocombined(dirlist)
if npts==ionospec.Param_List.shape[-1]:
tau,acfin=spect2acf(ionospec.Param_Names,ionospec.Param_List)
nloc,ntimes=acfin.shape[:2]
ambmat=RSTO.simparams['amb_dict']['WttMatrix']
np=ambmat.shape[0]
acfin_amb=sp.zeros((nloc,ntimes,np),dtype=acfin.dtype)
# get the original acf
ambmat=RSTO.simparams['amb_dict']['WttMatrix']
np=ambmat.shape[0]
for iloc,locarr in enumerate(acfin):
for itime,acfarr in enumerate(locarr):
acfin_amb[iloc,itime]=sp.dot(ambmat,acfarr)
acfin_amb=acfin_amb[:,0]
else:
acfin_amb=ionospec.Param_List[:,0]
if not os.path.isdir(costdir):
os.mkdir(costdir)
# pickle file stuff
pname=os.path.join(costdir,'cost{0}-{1}.pickle'.format(acfdir,invtype))
alpha_list=[]
errorlist=[]
errorlaglist=[]
datadiflist=[]
constlist=[]
if 'perryplane' in basedir.lower() or 'SimpData':
rbounds=[-500,500]
else:
rbounds=[0,500]
alpha_list_new=alpha_sweep.tolist()
for i in alpha_list:
if i in alpha_list_new:
alpha_list_new.remove(i)
for i in alpha_list_new:
ionoout,datadif,constdif=invertRSTO(RSTO,ionoin,alpha_list=i,invtype=invtype,rbounds=rbounds,Nlin=1)
datadiflist.append(datadif)
constlist.append(constdif)
acfout=ionoout.Param_List[:,0]
alpha_list.append(i)
outdata=sp.power(sp.absolute(acfout-acfin_amb),2)
aveerror=sp.sqrt(sp.nanmean(outdata,axis=0))
errorlaglist.append(aveerror)
errorlist.append(sp.nansum(aveerror))
pickleFile = open(pname, 'wb')
pickle.dump([alpha_list,errorlist,datadiflist,constlist,errorlaglist],pickleFile)
pickleFile.close()
mkalphalist(pname)
alphaarr=sp.array(alpha_list)
errorarr=sp.array(errorlist)
errorlagarr=sp.array(errorlaglist)
datadif=sp.array(datadiflist)
constdif=sp.array(constlist)
fig,axlist,axmain=plotalphaerror(alphaarr,errorarr,errorlagarr)
fig.savefig(os.path.join(costdir,'cost{0}-{1}.png'.format(acfdir,invtype)))
fig,axlist=plotLcurve(alphaarr,datadif,constdif)
fig.savefig(os.path.join(costdir,'lcurve{0}-{1}.png'.format(acfdir,invtype)))
def makehistdata(params,maindir):
""" This will make the histogram data for the statistics.
Inputs
params - A list of parameters that will have statistics created
maindir - The directory that the simulation data is held.
Outputs
datadict - A dictionary with the data values in numpy arrays. The keys are param names.
errordict - A dictionary with the data values in numpy arrays. The keys are param names.
errdictrel - A dictionary with the error values in numpy arrays, normalized by the correct value. The keys are param names.
"""
maindir=Path(maindir)
ffit = maindir.joinpath('Fitted','fitteddata.h5')
inputfiledir = maindir.joinpath('Origparams')
paramslower = [ip.lower() for ip in params]
eparamslower = ['n'+ip.lower() for ip in params]
# set up data dictionary
errordict = {ip:[] for ip in params}
errordictrel = {ip:[] for ip in params}
#Read in fitted data
Ionofit = IonoContainer.readh5(ffit)
times=Ionofit.Time_Vector
dataloc = Ionofit.Sphere_Coords
pnames = Ionofit.Param_Names
pnameslower = sp.array([ip.lower() for ip in pnames.flatten()])
p2fit = [sp.argwhere(ip==pnameslower)[0][0] if ip in pnameslower else None for ip in paramslower]
datadict = {ip:Ionofit.Param_List[:,:,p2fit[ipn]].flatten() for ipn, ip in enumerate(params)}
ep2fit = [sp.argwhere(ip==pnameslower)[0][0] if ip in pnameslower else None for ip in eparamslower]
edatadict = {ip:Ionofit.Param_List[:,:,ep2fit[ipn]].flatten() for ipn, ip in enumerate(params)}
# Determine which input files are to be used.
dirlist = [str(i) for i in inputfiledir.glob('*.h5')]
sortlist,outime,filelisting,timebeg,timelist_s = IonoContainer.gettimes(dirlist)
time2files = []
for itn,itime in enumerate(times):
log1 = (outime[:,0]>=itime[0]) & (outime[:,0]<itime[1])
log2 = (outime[:,1]>itime[0]) & (outime[:,1]<=itime[1])
log3 = (outime[:,0]<=itime[0]) & (outime[:,1]>itime[1])
tempindx = sp.where(log1|log2|log3)[0]
time2files.append(filelisting[tempindx])
curfilenum=-1
for iparam,pname in enumerate(params):
curparm = paramslower[iparam]
# Use Ne from input to compare the ne derived from the power.
if curparm == 'nepow':
curparm = 'ne'
datalist = []
for itn,itime in enumerate(times):
for iplot,filenum in enumerate(time2files[itn]):
filenum = int(filenum)
if curfilenum!=filenum:
curfilenum=filenum
datafilename = dirlist[filenum]
Ionoin = IonoContainer.readh5(datafilename)
if ('ti' in paramslower) or ('vi' in paramslower):
Ionoin = maketi(Ionoin)
pnames = Ionoin.Param_Names
pnameslowerin = sp.array([ip.lower() for ip in pnames.flatten()])
prmloc = sp.argwhere(curparm==pnameslowerin)
if prmloc.size !=0:
curprm = prmloc[0][0]
# build up parameter vector bs the range values by finding the closest point in space in the input
curdata = sp.zeros(len(dataloc))
for irngn, curcoord in enumerate(dataloc):
tempin = Ionoin.getclosestsphere(curcoord,[itime])[0]
Ntloc = tempin.shape[0]
tempin = sp.reshape(tempin,(Ntloc,len(pnameslowerin)))
curdata[irngn] = tempin[0,curprm]
datalist.append(curdata)
errordict[pname] = datadict[pname]-sp.hstack(datalist)
errordictrel[pname] = 100.*errordict[pname]/sp.absolute(sp.hstack(datalist))
return datadict,errordict,errordictrel,edatadict
def fitdata(basedir,configfile,optinputs):
""" This function will run the fitter on the estimated ACFs saved in h5 files.
Inputs:
basedir: A string for the directory that will hold all of the data for the simulation.
configfile: The configuration file for the simulation.
optinputs:A string that helps determine the what type of acfs will be fitted.
"""
# determine the input folders which can be ACFs from the full simulation
dirdict = {'fitting':('ACF', 'Fitted'), 'fittingmat':('ACFMat', 'FittedMat'),
'fittinginv':('ACFInv', 'FittedInv'), 'fittingmatinv':('ACFMatInv', 'FittedMatInv')}
dirio = dirdict[optinputs[0]]
inputdir = basedir/dirio[0]
outputdir = basedir/dirio[1]
fitlist = optinputs[1]
if len(optinputs) > 2:
exstr = optinputs[2]
printlines = optinputs[3]
else:
exstr = ''
dirlist = [str(i) for i in inputdir.glob('*lags{0}.h5'.format(exstr))]
dirlistsig = [str(i) for i in inputdir.glob('*sigs{0}.h5'.format(exstr))]
Ionoin = IonoContainer.readh5(dirlist[0])
if len(dirlistsig) == 0:
Ionoinsig = None
else:
Ionoinsig = IonoContainer.readh5(dirlistsig[0])
fitterone = Fitterionoconainer(Ionoin, Ionoinsig, configfile)
fitoutput = fitterone.fitdata(specfuncs.ISRSfitfunction,
fitterone.simparams['startfile'], fittimes=fitlist,
printlines=printlines)
#ipdb.set_trace()
(fitteddata, fittederror, funcevals, fittedcov) = fitoutput
if fitterone.simparams['Pulsetype'].lower() == 'barker':
paramlist = fitteddata
species = fitterone.simparams['species']
paramnames = ['Ne']
if not fittederror is None:
fittederronly = sp.sqrt(fittederror)
paramlist = sp.concatenate([fitteddata, fittederronly], axis=2)
paramnamese = ['n'+ip for ip in paramnames]
paranamsf = sp.array(paramnames+paramnamese)
else:
paranamsf = sp.array(paramnames)
else:
fittederronly = sp.sqrt(fittederror)
paramnames = []
species = fitterone.simparams['species']
# Seperate Ti and put it in as an element of the ionocontainer.
Ti = fitteddata[:, :, 1]
nTi = fittederronly[:, :, 1]
nTiTe = fittedcov[:, :, 0, 1]
nTiNe = fittedcov[:, :, 0, 2]
nTiVi = fittedcov[:, :, 0, 3]
nTeNe = fittedcov[:, :, 1, 2]
nTeVi = fittedcov[:, :, 1, 3]
nNeVi = fittedcov[:, :, 2, 3]
cov_list = [nTiTe[:, :, sp.newaxis], nTiNe[:, :, sp.newaxis],
nTiVi[:, :, sp.newaxis], nTeNe[:, :, sp.newaxis],
nTeVi[:, :, sp.newaxis], nNeVi[:, :, sp.newaxis]]
cov_list_names = ['nTiTe', 'nTiNe', 'nTiVi', 'nTeNe', 'nTeVi','nNeVi']
paramlist = sp.concatenate([fitteddata, Ti[:, :, sp.newaxis], fittederronly,
nTi[:, :, sp.newaxis], funcevals[:, :, sp.newaxis]]
+ cov_list, axis=2)
for isp in species[:-1]:
paramnames.append('Ni_'+isp)
paramnames.append('Ti_'+isp)
paramnames = paramnames+['Ne', 'Te', 'Vi', 'Nepow', 'Ti']
paramnamese = ['n'+ip for ip in paramnames]
paranamsf = sp.array(paramnames+paramnamese+['FuncEvals']+cov_list_names)
if fitlist is None:
timevec = Ionoin.Time_Vector
else:
if len(fitlist) == 0:
timevec = Ionoin.Time_Vector
else:
timevec = Ionoin.Time_Vector[fitlist]
# This requires
if set(Ionoin.Coord_Vecs) == {'x', 'y', 'z'}:
newver = 0
ionoout = IonoContainer(Ionoin.Cart_Coords, paramlist.real, timevec, ver=newver,
coordvecs=Ionoin.Coord_Vecs, paramnames=paranamsf,
species=species)
elif set(Ionoin.Coord_Vecs) == {'r', 'theta', 'phi'}:
newver = 1
ionoout = IonoContainer(Ionoin.Sphere_Coords, paramlist.real, timevec, ver=newver,
coordvecs=Ionoin.Coord_Vecs, paramnames=paranamsf,
species=species)
outfile = outputdir.joinpath('fitteddata{0}.h5'.format(exstr))
ionoout.saveh5(str(outfile))
def __init__(self, Ionodict, inifile, outdir, outfilelist=None):
"""
This function will create an instance of the RadarData class. It will
take in the values and create the class and make raw IQ data.
Inputs:
sensdict - A dictionary of sensor parameters
angles - A list of tuples which the first position is the az angle
and the second position is the el angle.
IPP - The interpulse period in seconds represented as a float.
Tint - The integration time in seconds as a float. This will be the
integration time of all of the beams.
time_lim - The length of time of the simulation the number of time points
will be calculated.
pulse - A numpy array that represents the pulse shape.
rng_lims - A numpy array of length 2 that holds the min and max range
that the radar will cover.
"""
(sensdict, simparams) = readconfigfile(inifile)
self.simparams = simparams
N_angles = len(self.simparams['angles'])
NNs = int(self.simparams['NNs'])
self.sensdict = sensdict
Npall = sp.floor(self.simparams['TimeLim'] / self.simparams['IPP'])
Npall = int(sp.floor(Npall / N_angles) * N_angles)
Np = Npall / N_angles
print("All spectrums created already")
filetimes = Ionodict.keys()
filetimes.sort()
ftimes = sp.array(filetimes)
simdtype = self.simparams['dtype']
pulsetimes = sp.arange(Npall) * self.simparams['IPP'] + ftimes.min()
pulsefile = sp.array(
[sp.where(itimes - ftimes >= 0)[0][-1] for itimes in pulsetimes])
# differentiate between phased arrays and dish antennas
if sensdict['Name'].lower() in ['risr', 'pfisr', 'risr-n']:
beams = sp.tile(sp.arange(N_angles), Npall / N_angles)
else:
# for dish arrays
brate = simparams['beamrate']
beams2 = sp.repeat(sp.arange(N_angles), brate)
beam3 = sp.concatenate((beams2, beams2[::-1]))
ntile = int(sp.ceil(Npall / len(beam3)))
leftover = int(Npall - ntile * len(beam3))
if ntile > 0:
beams = sp.tile(beam3, ntile)
beams = sp.concatenate((beams, beam3[:leftover]))
else:
beams = beam3[:leftover]
pulsen = sp.repeat(sp.arange(Np), N_angles)
pt_list = []
pb_list = []
pn_list = []
fname_list = []
self.datadir = outdir
self.maindir = outdir.parent
self.procdir = self.maindir / 'ACF'
Nf = len(filetimes)
progstr = 'Data from {:d} of {:d} being processed Name: {:s}.'
if outfilelist is None:
print('\nData Now being created.')
Noisepwr = v_Boltz * sensdict['Tsys'] * sensdict['BandWidth']
self.outfilelist = []
for ifn, ifilet in enumerate(filetimes):
outdict = {}
ifile = Ionodict[ifilet]
ifilename = Path(ifile).name
update_progress(
float(ifn) / Nf, progstr.format(ifn, Nf, ifilename))
curcontainer = IonoContainer.readh5(ifile)
if ifn == 0:
self.timeoffset = curcontainer.Time_Vector[0, 0]
pnts = pulsefile == ifn
pt = pulsetimes[pnts]
pb = beams[pnts]
pn = pulsen[pnts].astype(int)
rawdata = self.__makeTime__(pt, curcontainer.Time_Vector,
curcontainer.Sphere_Coords,
curcontainer.Param_List, pb)
d_shape = rawdata.shape
n_tempr = sp.random.randn(*d_shape).astype(simdtype)
n_tempi = 1j * sp.random.randn(*d_shape).astype(simdtype)
noise = sp.sqrt(Noisepwr / 2) * (n_tempr + n_tempi)
outdict['AddedNoise'] = noise
outdict['RawData'] = rawdata + noise
outdict['RawDatanonoise'] = rawdata
outdict['NoiseData'] = sp.sqrt(Noisepwr / 2) * (
sp.random.randn(len(pn), NNs).astype(simdtype) +
1j * sp.random.randn(len(pn), NNs).astype(simdtype))
outdict['Pulses'] = pn
outdict['Beams'] = pb
outdict['Time'] = pt
fname = '{0:d} RawData.h5'.format(ifn)
newfn = self.datadir / fname
self.outfilelist.append(str(newfn))
dict2h5(str(newfn), outdict)
#Listing info
pt_list.append(pt)
pb_list.append(pb)
pn_list.append(pn)
fname_list.append(fname)
infodict = {
'Files': fname_list,
'Time': pt_list,
'Beams': pb_list,
'Pulses': pn_list
}
dict2h5(str(outdir.joinpath('INFO.h5')), infodict)
else:
infodict = h52dict(str(outdir.joinpath('INFO.h5')))
alltime = sp.hstack(infodict['Time'])
self.timeoffset = alltime.min()
self.outfilelist = outfilelist
def plotacf(testdir,imgdir,wtimes=False,acfpath='ACFInv',lagfile='00lags.h5',filetemplate=None,lag=0):
"""
This will plot all of the fitted data with each time step as a pcolor images of
electron density and electron density from power mesurements.
Inputs
testdir - The directory with the input data in h5 files formated for the ionocontainer structure.
imgdir - The directory that holds the images.
"""
# if os.path.exists(imgdir):
# imgfiles = glob.glob(os.path.join(imgdir,'*.png'))
# for imgf in imgfiles:
# os.remove(imgf)
if not os.path.exists(imgdir):
os.mkdir(imgdir)
filename = os.path.join(testdir,acfpath,lagfile)
iono = IonoContainer.readh5(filename)
# if iono.Param_List.shape[-1]==1:
# nemax=1e10
# nemin=0
nt = iono.Time_Vector.shape[0]
if set(iono.Coord_Vecs)=={'r','theta','phi'}:
rngrdr =iono.Sphere_Coords[:,0].astype('float32')
sign1 = sp.sign(iono.Sphere_Coords[:,1])
el = iono.Sphere_Coords[:,2].astype('float32')
elvec,elinv = sp.unique(el,return_inverse=True)
nbeams = len(elvec)
nrg = len(rngrdr)/nbeams
Rngrdrmat = sp.reshape(rngrdr,(nrg,nbeams))
Signmat = sp.reshape(sign1,(nrg,nbeams))
Elmat = sp.reshape(el,(nrg,nbeams))
Xmat = Rngrdrmat*Signmat*sp.cos(Elmat*sp.pi/180.)
Zmat = Rngrdrmat*sp.sin(Elmat*sp.pi/180.)
Ne = iono.Param_List[:,:,lag].reshape(nrg,nbeams,nt).real*ne_red
elif set(iono.Coord_Vecs)=={'x','y','z'}:
rng = sp.sqrt(iono.Cart_Coords[:,0]**2+iono.Cart_Coords[:,1]**2)*sp.sign(iono.Cart_Coords[:,1]).astype('float32')
z = iono.Cart_Coords[:,2].astype('float32')
rngvec = sp.unique(rng)
zvec = sp.unique(z)
Xmat = rng.reshape(len(zvec),len(rngvec))
Zmat = z.reshape(len(zvec),len(rngvec))
Ne = iono.Param_List[:,:,lag].reshape(len(zvec),len(rngvec),nt).real*ne_red
imcount=0
if filetemplate is None:
filetemplate = 'acflag{0}'.format(lag)
dsetname = os.path.split(os.path.dirname(testdir))[-1]
print "Plotting Output data for "+dsetname
if 'perryplane' in testdir.lower():
xlim = [-200.,360.]
xticks = [-150.,0.,150.,300.]
else:
xlim = [0.,400.]
xticks = [0.,150.,300]
ncols=1
figsize = (5,7)
ylim = [100.,500]
for itimen,itime in enumerate(iono.Time_Vector):
print "{0} Output for {1} of {2}".format(dsetname,itimen,len(iono.Time_Vector))
Nemat = Ne[:,:,itimen]
fig ,axmat= plt.subplots(nrows=1,ncols=ncols,facecolor='w',figsize=figsize,sharey=True)
avec=[axmat]
plt.sca(avec[0])
avec[0].set_xlabel('X Plane in km',fontsize=fs)
avec[0].set_ylabel('Alt in km',fontsize=fs)
pc1 = avec[0].pcolor(Xmat,Zmat,Nemat,cmap = defmap,vmin=nemin,vmax=nemax)
plt.tick_params(labelsize=16)
plt.xticks(xticks)
avec[0].set_xlim(xlim)
avec[0].set_ylim(ylim)
avec[0].set_title('',fontsize=fs)
tick_locator = ticker.MaxNLocator(nbins=5)
cb1 = plt.colorbar(pc1, ax=avec[0])
cb1.ax.xaxis.set_label_position('top')
cb1.ax.set_xlabel(necbarstr)
cb1.locator = tick_locator
cb1.update_ticks()
plt.tight_layout()
fname= '{0:0>3}_'.format(imcount)+filetemplate+'.png'
plt.savefig(os.path.join(imgdir,fname),dpi=300)
imcount=imcount+1
plt.close(fig)
def plotpercenterror(testdir,imgdir,config,wtimes=False,fitpath='Fitted',fitfile='fitteddata.h5'):
"""
This will plot all of the fitted data with each time step as a pcolor images of
electron density and electron density from power mesurements.
Inputs
testdir - The directory with the input data in h5 files formated for the ionocontainer structure.
imgdir - The directory that holds the images.
"""
(sensdict,simparams)=readconfigfile(config)
tvec = simparams['Timevec']
filetemplate = 'fitteddataerrorpercent'
if os.path.exists(imgdir):
imgfiles = glob.glob(os.path.join(imgdir,'*'+filetemplate+'.png'))
for imgf in imgfiles:
os.remove(imgf)
else:
os.mkdir(imgdir)
filename = os.path.join(testdir,fitpath,fitfile)
iono = IonoContainer.readh5(filename)
Iono1 = GeoData(readIono,[iono])
rngrdr =Iono1.dataloc[:,0].astype('float32')
sign1 = sp.sign(Iono1.dataloc[:,1])
el = Iono1.dataloc[:,2].astype('float32')
elvec,elinv = sp.unique(el,return_inverse=True)
nbeams = len(elvec)
nrg = len(rngrdr)/nbeams
nt = Iono1.times.shape[0]
Rngrdrmat = sp.reshape(rngrdr,(nrg,nbeams))
Signmat = sp.reshape(sign1,(nrg,nbeams))
Elmat = sp.reshape(el,(nrg,nbeams))
permin,permax=[0.,100.]
Xmat = Rngrdrmat*Signmat*sp.cos(Elmat*sp.pi/180.)
Zmat = Rngrdrmat*sp.sin(Elmat*sp.pi/180.)
Ne = Iono1.data['Ne'].reshape(nrg,nbeams,nt)
Ti = Iono1.data['Ti'].reshape(nrg,nbeams,nt)
Te = Iono1.data['Te'].reshape(nrg,nbeams,nt)
nNe = 100.*Iono1.data['nNe'].reshape(nrg,nbeams,nt)/Ne
nTe = 100.*Iono1.data['nTe'].reshape(nrg,nbeams,nt)/Te
nTi = 100.*Iono1.data['nTi'].reshape(nrg,nbeams,nt)/Ti
imcount=0
dsetname = os.path.split(os.path.dirname(testdir))[-1]
print "Plotting Output error data for "+dsetname
if 'perryplane' in testdir.lower():
xlim = [-200.,360.]
xticks = [-150.,0.,150.,300.]
allparams=True
ncols=3
figsize = (15,7)
else:
xlim = [0.,400.]
xticks = [0.,150.,300]
allparams = False
ncols=1
figsize = (5,7)
ylim = [100.,500]
for itimen,itime in enumerate(Iono1.times):
print "{0} Output for {1} of {2}".format(dsetname,itimen,len(Iono1.times))
Nemat = nNe[:,:,itimen]
Timat = nTi[:,:,itimen]
Temat = nTe[:,:,itimen]
fig ,axmat= plt.subplots(nrows=1,ncols=ncols,facecolor='w',figsize=figsize,sharey=True)
if allparams:
avec=axmat.flatten()
else:
avec=[axmat]
plt.sca(avec[0])
avec[0].set_xlabel('X Plane in km',fontsize=fs)
avec[0].set_ylabel('Alt in km',fontsize=fs)
pc1 = avec[0].pcolor(Xmat,Zmat,Nemat,cmap = defmap,vmin=permin,vmax=permax)
plt.tick_params(labelsize=16)
plt.xticks(xticks)
avec[0].set_xlim(xlim)
avec[0].set_ylim(ylim)
avec[0].set_title('Electron Density % Error',fontsize=fs)
tick_locator = ticker.MaxNLocator(nbins=5)
cb1 = plt.colorbar(pc1, ax=avec[0])
cb1.ax.xaxis.set_label_position('top')
cb1.ax.set_xlabel(ercbarstr,fontsize=fscb)
cb1.locator = tick_locator
cb1.update_ticks()
if allparams:
plt.sca(avec[1])
plt.tick_params(labelsize=16)
plt.xticks(xticks)
avec[1].set_xlabel('X Plane in km',fontsize=fs)
pc2 = avec[1].pcolor(Xmat,Zmat,Temat,cmap = defmap,vmin=permin,vmax=permax)
avec[1].set_xlim(xlim)
avec[1].set_ylim(ylim)
avec[1].set_title('Electron Temperature % Error',fontsize=fs)
cb2 = plt.colorbar(pc2, ax=avec[1])
cb2.ax.xaxis.set_label_position('top')
cb2.ax.set_xlabel(ercbarstr,fontsize=fscb)
cb2.locator = tick_locator
cb2.update_ticks()
plt.sca(avec[2])
plt.xticks(xticks)
plt.tick_params(labelsize=16)
avec[2].set_xlabel('X Plane in km',fontsize=fs)
pc3 = avec[2].pcolor(Xmat,Zmat,Timat,cmap = defmap,vmin=permin,vmax=permax)
avec[2].set_xlim(xlim)
avec[2].set_ylim(ylim)
avec[2].set_title('Ion Temperature % Error',fontsize=fs)
cb3 = plt.colorbar(pc3, ax=avec[2])
cb3.ax.xaxis.set_label_position('top')
cb3.ax.set_xlabel(ercbarstr,fontsize=fscb)
cb3.locator = tick_locator
cb3.update_ticks()
plt.tight_layout()
if wtimes:
plt.subplots_adjust(top=0.9)
spti = fig.suptitle('Percent Error at {0} seconds'.format(int(tvec[itimen])),fontsize=24)
fname= '{0:0>3}_'.format(imcount)+filetemplate+'.png'
plt.savefig(os.path.join(imgdir,fname),dpi=300)
imcount=imcount+1
plt.close(fig)
def plotinputdata(testdir,imgdir,wtimes=False):
"""
This will plot all of the input data with each time step as a pcolor images of
electron density and ion tempreture.
Inputs
testdir - The directory with the input data in h5 files formated for the ionocontainer structure.
imgdir - The directory that holds the images.
"""
if os.path.exists(imgdir):
imgfiles = glob.glob(os.path.join(imgdir,'*.png'))
for imgf in imgfiles:
os.remove(imgf)
else:
os.mkdir(imgdir)
filelist = glob.glob(os.path.join(testdir,'Origparams','*.h5'))
numlist = [os.path.splitext(os.path.split(x)[-1])[0] for x in filelist]
numdict = {numlist[i]:filelist[i] for i in range(len(filelist))}
slist = sorted(numlist,key=ke)
imcount = 0
filetemplate = 'inputdata'
dsetname = os.path.split(os.path.dirname(testdir))[-1]
print "Plotting input data for "+dsetname
if 'perryplane' in testdir.lower():
xlim = [-200.,360.]
xticks = [-150.,0.,150.,300.]
allparams = True
ncols=3
figsize = (15,7)
else:
xlim = [0.,400.]
xticks = [0.,150.,300.]
allparams = False
ncols=1
figsize = (5,7)
ylim = [100.,500.]
f1 = True
for inumn, inum in enumerate(slist):
print "{0} Input for {1} of {2}".format(dsetname,inumn,len(slist))
ifile = numdict[inum]
iono = IonoContainer.readh5(ifile)
Iono1 = GeoData(readIono,[iono])
nt = Iono1.times.shape[0]
rng = sp.sqrt(Iono1.dataloc[:,0]**2+Iono1.dataloc[:,1]**2)*sp.sign(Iono1.dataloc[:,1])
z = Iono1.dataloc[:,2]
rngvec = sp.unique(rng)
zvec = sp.unique(z)
rngmat = rng.reshape(len(zvec),len(rngvec))
zmat = z.reshape(len(zvec),len(rngvec))
Ne = Iono1.data['Ne'].reshape(len(zvec),len(rngvec),nt)*ne_red
Ti = Iono1.data['Ti'].reshape(len(zvec),len(rngvec),nt)
Te = Iono1.data['Te'].reshape(len(zvec),len(rngvec),nt)
for itimen,itime in enumerate(Iono1.times):
if f1:
f1=False
t0 = itime[0]
fig ,axmat= plt.subplots(nrows=1,ncols=ncols,facecolor='w',figsize=figsize ,sharey=True)
if allparams:
avec = axmat.flatten()
else:
avec =[axmat]
plt.sca(avec[0])
plt.xticks(xticks)
plt.tick_params(labelsize=16)
avec[0].set_xlabel('X Plane in km',fontsize=fs)
avec[0].set_ylabel('Alt in km',fontsize=fs)
pc1 = avec[0].pcolor(rngmat,zmat,Ne[:,:,itimen],cmap = defmap,vmin=nemin,vmax=nemax)
avec[0].set_xlim(xlim)
avec[0].set_ylim(ylim)
avec[0].set_title('Electron Density',fontsize=fs)
#pc1.set_norm(colors.LogNorm(vmin=5e8,vmax=5e12))
tick_locator = ticker.MaxNLocator(nbins=5)
cb1 = plt.colorbar(pc1, ax=avec[0])
cb1.ax.xaxis.set_label_position('top')
cb1.ax.set_xlabel(necbarstr,fontsize=fscb)
cb1.locator = tick_locator
cb1.update_ticks()
if allparams:
plt.sca(avec[1])
plt.xticks(xticks)
plt.tick_params(labelsize=16)
avec[1].set_xlabel('X Plane in km',fontsize=fs)
pc2 = avec[1].pcolor(rngmat,zmat,Te[:,:,itimen],cmap = defmap,vmin=temin,vmax=temax)
avec[1].set_xlim(xlim)
avec[1].set_ylim(ylim)
avec[1].set_title('Electron Temperature',fontsize=fs)
cb2 = plt.colorbar(pc2, ax=avec[1])
cb2.ax.xaxis.set_label_position('top')
cb2.ax.set_xlabel(tecbarstr,fontsize=fscb)
cb2.locator = tick_locator
cb2.update_ticks()
plt.sca(avec[2])
plt.xticks(xticks)
plt.tick_params(labelsize=16)
avec[2].set_xlabel('X Plane in km',fontsize=fs)
pc3 = avec[2].pcolor(rngmat,zmat,Ti[:,:,itimen],cmap = defmap,vmin=timin,vmax=timax)
avec[2].set_xlim(xlim)
avec[2].set_ylim(ylim)
avec[2].set_title('Ion Temperature',fontsize=fs)
cb3 = plt.colorbar(pc3, ax=avec[2])
cb3.ax.xaxis.set_label_position('top')
cb3.ax.set_xlabel(ticbarstr,fontsize=fscb)
cb3.locator = tick_locator
cb3.update_ticks()
plt.tight_layout()
if wtimes:
plt.subplots_adjust(top=0.9)
spti = fig.suptitle('Parameters at {0} seconds'.format(int(itime[0]-t0)),fontsize=24)
fname= '{0:0>3}_'.format(imcount)+filetemplate+'.png'
# for ax in avec:
# for label in (ax.get_xticklabels() + ax.get_yticklabels()):
# label.set_fontsize(20)
imcount=imcount+1
plt.savefig(os.path.join(imgdir,fname),dpi=200)
plt.close(fig)
def lagdict2ionocont(DataLags, NoiseLags, sensdict, simparams, time_vec):
"""This function will take the data and noise lags and create an instance of the
Ionocontanier class. This function will also apply the summation rule to the lags.
Inputs
DataLags - A dictionary """
# Pull in Location Data
angles = simparams['angles']
ang_data = sp.array([[iout[0], iout[1]] for iout in angles])
rng_vec = simparams['Rangegates']
n_samps = len(rng_vec)
# pull in other data
pulse = simparams['Pulse']
p_samps = len(pulse)
pulsewidth = p_samps * sensdict['t_s']
txpower = sensdict['Pt']
if sensdict['Name'].lower() in ['risr', 'pfisr', 'risr-n']:
Ksysvec = sensdict['Ksys']
else:
beamlistlist = sp.array(simparams['outangles']).astype(int)
inplist = sp.array([i[0] for i in beamlistlist])
Ksysvec = sensdict['Ksys'][inplist]
ang_data_temp = ang_data.copy()
ang_data = sp.array(
[ang_data_temp[i].mean(axis=0) for i in beamlistlist])
sumrule = simparams['SUMRULE']
rng_vec2 = simparams['Rangegatesfinal']
Nrng2 = len(rng_vec2)
minrg = p_samps - 1 + sumrule[0].min()
maxrg = Nrng2 + minrg
# Copy the lags
lagsData = DataLags['ACF'].copy()
# Set up the constants for the lags so they are now
# in terms of density fluxtuations.
angtile = sp.tile(ang_data, (Nrng2, 1))
rng_rep = sp.repeat(rng_vec2, ang_data.shape[0], axis=0)
coordlist = sp.zeros((len(rng_rep), 3))
[coordlist[:, 0], coordlist[:, 1:]] = [rng_rep, angtile]
(Nt, Nbeams, Nrng, Nlags) = lagsData.shape
# make a range average to equalize out the conntributions from each gate
plen2 = int(sp.floor(float(p_samps - 1) / 2))
samps = sp.arange(0, p_samps, dtype=int)
rng_ave = sp.zeros((Nrng, p_samps))
for isamp in range(plen2, Nrng + plen2):
for ilag in range(p_samps):
toplag = int(sp.floor(float(ilag) / 2))
blag = int(sp.ceil(float(ilag) / 2))
if toplag == 0:
sampsred = samps[blag:]
else:
sampsred = samps[blag:-toplag]
cursamps = isamp - sampsred
keepsamps = sp.logical_and(cursamps >= 0, cursamps < Nrng)
cursamps = cursamps[keepsamps]
rng_samps = rng_vec[cursamps]**2 * 1e6
keepsamps2 = rng_samps > 0
if keepsamps2.sum() == 0:
continue
rng_samps = rng_samps[keepsamps2]
rng_ave[isamp - plen2, ilag] = 1. / (sp.mean(1. / (rng_samps)))
rng_ave_temp = rng_ave.copy()
if simparams['Pulsetype'].lower() is 'barker':
rng_ave_temp = rng_ave[:, 0][:, sp.newaxis]
# rng_ave = rng_ave[int(sp.floor(plen2)):-int(sp.ceil(plen2))]
# rng_ave = rng_ave[minrg:maxrg]
rng3d = sp.tile(rng_ave_temp[sp.newaxis, sp.newaxis, :, :],
(Nt, Nbeams, 1, 1))
ksys3d = sp.tile(Ksysvec[sp.newaxis, :, sp.newaxis, sp.newaxis],
(Nt, 1, Nrng, Nlags))
# rng3d = sp.tile(rng_ave[:, sp.newaxis, sp.newaxis, sp.newaxis], (1, Nlags, Nt, Nbeams))
# ksys3d = sp.tile(Ksysvec[sp.newaxis, sp.newaxis, sp.newaxis, :], (Nrng2, Nlags, Nt, 1))
radar2acfmult = rng3d / (pulsewidth * txpower * ksys3d)
pulses = sp.tile(DataLags['Pulses'][:, :, sp.newaxis, sp.newaxis],
(1, 1, Nrng, Nlags))
time_vec = time_vec[:Nt]
# Divid lags by number of pulses
lagsData = lagsData / pulses
# Set up the noise lags and divid out the noise.
lagsNoise = NoiseLags['ACF'].copy()
lagsNoise = sp.mean(lagsNoise, axis=2)
pulsesnoise = sp.tile(NoiseLags['Pulses'][:, :, sp.newaxis], (1, 1, Nlags))
lagsNoise = lagsNoise / pulsesnoise
lagsNoise = sp.tile(lagsNoise[:, :, sp.newaxis, :], (1, 1, Nrng, 1))
# multiply the data and the sigma by inverse of the scaling from the radar
lagsData = lagsData * radar2acfmult
lagsNoise = lagsNoise * radar2acfmult
# Apply summation rule
# lags transposed from (time,beams,range,lag)to (range,lag,time,beams)
lagsData = sp.transpose(lagsData, axes=(2, 3, 0, 1))
lagsNoise = sp.transpose(lagsNoise, axes=(2, 3, 0, 1))
lagsDatasum = sp.zeros((Nrng2, Nlags, Nt, Nbeams), dtype=lagsData.dtype)
lagsNoisesum = sp.zeros((Nrng2, Nlags, Nt, Nbeams), dtype=lagsNoise.dtype)
for irngnew, irng in enumerate(sp.arange(minrg, maxrg)):
for ilag in range(Nlags):
lsumtemp = lagsData[irng + sumrule[0, ilag]:irng +
sumrule[1, ilag] + 1, ilag].sum(axis=0)
lagsDatasum[irngnew, ilag] = lsumtemp
nsumtemp = lagsNoise[irng + sumrule[0, ilag]:irng +
sumrule[1, ilag] + 1, ilag].sum(axis=0)
lagsNoisesum[irngnew, ilag] = nsumtemp
# subtract out noise lags
lagsDatasum = lagsDatasum - lagsNoisesum
# Put everything in a parameter list
Paramdata = sp.zeros((Nbeams * Nrng2, Nt, Nlags), dtype=lagsData.dtype)
# Put everything in a parameter list
# transpose from (range,lag,time,beams) to (beams,range,time,lag)
# lagsDatasum = lagsDatasum*radar2acfmult
# lagsNoisesum = lagsNoisesum*radar2acfmult
lagsDatasum = sp.transpose(lagsDatasum, axes=(3, 0, 2, 1))
lagsNoisesum = sp.transpose(lagsNoisesum, axes=(3, 0, 2, 1))
# multiply the data and the sigma by inverse of the scaling from the radar
# lagsDatasum = lagsDatasum*radar2acfmult
# lagsNoisesum = lagsNoisesum*radar2acfmult
# Calculate a variance using equation 2 from Hysell's 2008 paper. Done use full covariance matrix because assuming nearly diagonal.
# Get the covariance matrix
pulses_s = sp.transpose(pulses, axes=(1, 2, 0, 3))[:, :Nrng2]
Cttout = makeCovmat(lagsDatasum, lagsNoisesum, pulses_s, Nlags)
Paramdatasig = sp.zeros((Nbeams * Nrng2, Nt, Nlags, Nlags),
dtype=Cttout.dtype)
curloc = 0
for irng in range(Nrng2):
for ibeam in range(Nbeams):
Paramdata[curloc] = lagsDatasum[ibeam, irng].copy()
Paramdatasig[curloc] = Cttout[ibeam, irng].copy()
curloc += 1
ionodata = IonoContainer(coordlist,
Paramdata,
times=time_vec,
ver=1,
paramnames=sp.arange(Nlags) * sensdict['t_s'])
ionosigs = IonoContainer(
coordlist,
Paramdatasig,
times=time_vec,
ver=1,
paramnames=sp.arange(Nlags**2).reshape(Nlags, Nlags) * sensdict['t_s'])
return (ionodata, ionosigs)
Created on Tue Apr 14 18:06:56 2015
@author: John Swoboda
"""
import os, glob
from SimISR.IonoContainer import IonoContainer
from GeoData.GeoData import GeoData
from GeoData.utilityfuncs import readIono
if __name__ == "__main__":
inputdir = "/Users/Bodangles/Documents/Python/PhantomProc/Origparams"
outdir = "/Users/Bodangles/Documents/MATLAB/phantoms/Origparams"
dirlist = glob.glob(os.path.join(inputdir, "*.mat"))
keep = ["Ne", "Te", "Ti"]
for ifile in dirlist:
Ionoin = IonoContainer.readmat(ifile)
GD = GeoData(readIono, [Ionoin])
for ikey in GD.data.keys():
if ikey not in keep:
del GD.data[ikey]
fname = os.path.splitext(os.path.split(ifile)[1])[0]
GD.write_h5(os.path.join(outdir, fname + ".h5"))
print("Saved " + os.path.join(outdir, fname + ".h5"))
def fit2geodata(inputfile):
Ionoin = IonoContainer.readh5(inputfile)
keep = ["Ne", "Te", "Ti"]
GD = GeoData(readIono, [Ionoin])
for ikey in GD.data.keys():
if ikey not in keep:
def readmattsdata(filename,datadir,outdir,keepspec=[0,1,2,6],angle=20.5):
d2r=sp.pi/180.
angr=d2r*angle
lsp=7
# Read in Data
inst = sio.loadmat(os.path.join(datadir,filename))
xg=inst['xg'][0,0]
x1v = xg['xp']# added to avoid gratting lobes.
x3v = xg['zp']
[x1mat,x3mat] = sp.meshgrid(x1v,x3v);
E = x1mat*sp.sin(angr)#x
N = x1mat*sp.cos(angr)#y
U = x3mat
lxs=x3mat.size
Time_Vector = sp.column_stack([inst['t'],inst['t']+15])
ns =inst['ns']
print('Loaded densities...');
ns= sp.reshape(ns,[lxs,lsp])
Ts =inst['Ts']
print('Loaded temperatures...')
Ts=sp.reshape(Ts,[lxs,lsp])
vs = inst['vsx1']
print('Loaded parallel velocities...\n');
# derive velocity from ExB
Ez,Ex=sp.gradient(-1*inst['Phi'])
dx=sp.diff(xg['x'].flatten())[0]
dEdx=Ex/dx
vx1=-1*dEdx/xg['Bmag']
# from looking at the data it seems that the velocity is off by a factor of
# 10.
vx1=vx1.flatten()/10.
vs=sp.reshape(vs,[lxs,lsp])
vs=sp.sum(ns[:,:(lsp-1)]*vs[:,:(lsp-1)],1)/ns[:,lsp-1]
v_e= vx1*sp.sin(angr)
v_n = vx1*sp.cos(angr)
v_u = vs
#change units of velocity to km/s
Velocity = sp.reshape(sp.column_stack([v_e,v_n,v_u]),[lxs,1,3])
# reduce the number of spcecies
# if islogical(keepspec)
# keepspec(end)=true;
# keepspecnum = sum(keepspec);
# elseif ~any(keepspec==numspec)
# keepspec = [keepspec(:),numspec];
# keepspecnum = length(keepspec);
# else
# keepspecnum = length(keepspec);
# end
nsout = sp.reshape(ns[:,keepspec],[lxs,1,len(keepspec)])
Tsout = sp.reshape(Ts[:,keepspec],[lxs,1,len(keepspec)])
# Put everything in to ionocontainer structure
Cart_Coords = sp.column_stack([E.flatten(),N.flatten(),U.flatten()])*1e-3
Param_List = sp.concatenate((sp.expand_dims(nsout,nsout.ndim),sp.expand_dims(Tsout,Tsout.ndim)),-1);
Species = sp.array(['O+','NO+','N2+','O2+','N+', 'H+','e-'])
Species = Species[keepspec]
fpart=os.path.splitext(filename)[0]
fout=os.path.join(outdir,fpart+'.h5')
ionoout=IonoContainer(Cart_Coords,Param_List,Time_Vector,ver=0,species=Species,velocity=Velocity)
ionoout.saveh5(fout)
def mult_iono(self, ionoin_list):
"""
This will apply the forward model to the contents of an ionocontainer object. It is assuming that
this is an ionocontainer holding the spectra.
"""
ntout = self.Time_Out.shape[0]
nlout = self.Cart_Coords_Out.shape[0]
blist_in, blist_out = self.blocklocs
amb_dict = self.simparams['amb_dict']
ambmat = amb_dict['WttMatrix']
overlaps = self.overlaps
t_s = self.sensdict['t_s']
if isinstance(ionoin_list, list) or isinstance(ionoin_list, str):
Iono_in = makeionocombined(ionoin_list)
else:
Iono_in = ionoin_list
ionocart = Iono_in.Cart_Coords
if self.simparams['numpoints'] == Iono_in.Param_List.shape[-1]:
tau, acf = spect2acf(Iono_in.Param_Names, Iono_in.Param_List)
np = ambmat.shape[0]
else:
acf = Iono_in.Param_List
np = acf.shape[-1]
np_in = acf.shape[-1]
tau_out = t_s * np.arange(np)
outdata = np.zeros((nlout, ntout, np), dtype=acf.dtype)
assert np.allclose(
ionocart,
self.Cart_Coords_In), "Spatial Coordinates need to be the same"
for it_out in range(ntout):
overlists = overlaps[it_out]
irows = blist_out[it_out]
curintimes = [i[0] for i in overlists]
curintratio = [i[1] for i in overlists]
if self.mattype.lower() == 'sim':
curintimes = [curintimes[0]]
curintratio = [1.]
cur_outmat = self.RSTMat[irows[0]:irows[1], :]
icols = blist_in[it_out]
cur_mat = cur_outmat[:, icols[0]:icols[1]]
for i_it, it_in in enumerate(curintimes):
tempdata = np.zeros((np_in, nlout), dtype=acf.dtype)
for iparam in range(np_in):
tempdata[iparam] = cur_mat.dot(acf[:, it_in, iparam])
if self.simparams['numpoints'] == Iono_in.Param_List.shape[-1]:
tempdata = np.dot(ambmat, tempdata)
outdata[:, it_out] = np.transpose(
tempdata) * curintratio[i_it] + outdata[:, it_out]
outiono = IonoContainer(self.Sphere_Coords_Out,
outdata,
times=self.Time_Out,
sensor_loc=Iono_in.Sensor_loc,
ver=1,
coordvecs=['r', 'theta', 'phi'],
paramnames=tau_out)
return outiono
def plotgradlines(inputlist,fitiono,alt,times,paramlist=['Ne','Te','Ti']):
"""
Plots the values along a specific alittude.
"""
inputiono = makeionocombined(inputlist)
Iono1 = GeoData(readIono,[inputiono])
fitiono = IonoContainer.readh5(fitiono)
fitGeo = GeoData(readIono,[fitiono])
paramlist = ['Ne','Te','Ti']
(x,y,z) = inputiono.Cart_Coords.transpose()
r = sp.sqrt(x**2+y**2)*sp.sign(y)
incoords = sp.column_stack((r,z,sp.ones_like(z)))
ru,zu =[ sp.unique(r),sp.unique(z)]
Rmat,Zmat = sp.meshgrid(ru,zu)
zinput = sp.argmin(sp.absolute(zu-alt))
(xf,yf,zf) = fitiono.Cart_Coords.transpose()
rf = sp.sqrt(xf**2+yf**2)*sp.sign(yf)
outcoords = sp.column_stack((rf,zf,sp.ones_like(zf)))
rfu,zfu =[ sp.unique(rf),sp.unique(zf)]
Rfmat,Zfmat = sp.meshgrid(rfu,zfu)
zoutput = sp.argmin(sp.absolute(zfu-alt))
fitGeo.interpolate(incoords,Iono1.coordnames,method='linear',fill_value=sp.nan,twodinterp = True,oldcoords=outcoords)
uz = sp.unique(z)
ur = sp.unique(r)
(rmat,zmat) = sp.meshgrid(ur,uz)
inputdata = {}
for iparm in paramlist:
if iparm =='Nepow':
iparm='Ne'
curdata = Iono1.data[iparm][:,times[0]]
inputdata[iparm] = sp.reshape(curdata,Rmat.shape)[zinput]
outputdata = {}
for iparm in paramlist:
curdata = fitGeo.data[iparm][:, times[1]]
outputdata[iparm] = sp.reshape(curdata,Rmat.shape)[zinput]
fig, axvec = plt.subplots(len(paramlist),1,sharey=False,figsize=(12,5*len(paramlist)))
for ipn,iparam in enumerate(paramlist):
ax = axvec[ipn]
ax2 = ax.twinx()
p1, = ax.plot(ru,inputdata[iparam],'b-',label='In',linewidth=3)
p2, = ax.plot(ru,outputdata[iparam],'b--',label='Out',linewidth=3)
ax.set_title(iparam)
ax.set_xlabel('X Plane in km')
gi = sp.gradient(inputdata[iparam])/sp.gradient(ru)
go = sp.gradient(outputdata[iparam])/sp.gradient(ru)
p3, = ax2.plot(ru,gi,'g-',label='Grad In',linewidth=3)
p4, = ax2.plot(ru,go,'g--',label='Grad Out',linewidth=3)
ax.yaxis.label.set_color(p1.get_color())
ax2.yaxis.label.set_color(p3.get_color())
lines = [p1,p2,p3,p4]
ax.legend(lines,[l.get_label() for l in lines])
plt.tight_layout()
return(fig)
def plotsampling(testdir,outfile, ifile=None, wtimes=False):
"""This will plot all of the input data with each time step as a pcolor images of
electron density and ion temperature.
Inputs
testdir - The directory with the input data in h5 files formated for the ionocontainer structure.
imgdir - The directory that holds the images."""
imcount = 0
if ifile is None:
filelist = glob.glob(os.path.join(testdir,'Origparams','*.h5'))
numlist = [os.path.splitext(os.path.split(x)[-1])[0] for x in filelist]
numdict = {numlist[i]:filelist[i] for i in range(len(filelist))}
slist = sorted(numlist,key=ke)
inumn=0
inum = slist[0]
dsetname = os.path.split(os.path.dirname(testdir))[-1]
print "Plotting input data for "+dsetname
print "{0} Input for {1} of {2}".format(dsetname,inumn,len(slist))
ifile = numdict[inum]
iono = IonoContainer.readh5(ifile)
filetemplate = 'inputdata'
if 'perryplane' in testdir.lower():
xlim = [-150.,360.]
xticks = [0.,150.,300.]
allparams = True
ncols=3
figsize = (5,7)
else:
xlim = [0.,400.]
xticks = [0.,150.,300.]
allparams = False
ncols=1
figsize = (5,7)
ylim = [100.,500.]
f1 = True
if iono.Param_List.shape[-1]==1:
nemax=1e10*ne_red
nemin=0
iono2 = IonoContainer.readh5(outfile)
xout,yout,zout = iono2.Cart_Coords.transpose()
rout=sp.sqrt(xout**2+yout**2)
Iono1 = GeoData(readIono,[iono])
nt = Iono1.times.shape[0]
rng = sp.sqrt(Iono1.dataloc[:,0]**2+Iono1.dataloc[:,1]**2)*sp.sign(Iono1.dataloc[:,1]).astype('float32')
z = Iono1.dataloc[:,2].astype('float32')
rngvec = sp.unique(rng)
zvec = sp.unique(z)
rngmat = rng.reshape(len(zvec),len(rngvec))
zmat = z.reshape(len(zvec),len(rngvec))
if 'Ne' in Iono1.data.keys():
Ne = Iono1.data['Ne'].reshape(len(zvec),len(rngvec),nt)*ne_red
elif '0' in Iono1.data.keys():
Ne = Iono1.data['Ne'].reshape(len(zvec),len(rngvec),nt)*ne_red
itimen=0
itime = Iono1.times[0]
if f1:
f1=False
t0 = itime[0]
fig ,axmat= plt.subplots(nrows=1,ncols=ncols,facecolor='w',figsize=figsize ,sharey=True)
if allparams:
avec = axmat.flatten()
else:
avec =[axmat]
plt.sca(avec[0])
plt.xticks(xticks)
plt.tick_params(labelsize=16)
avec[0].set_xlabel('X Plane in km',fontsize=fs)
avec[0].set_ylabel('Alt in km',fontsize=fs)
pc1 = avec[0].pcolor(rngmat,zmat,Ne[:,:,itimen],cmap = defmap,vmin=nemin,vmax=nemax)
avec[0].set_xlim(xlim)
avec[0].set_ylim(ylim)
#pc1.set_norm(colors.LogNorm(vmin=5e8,vmax=5e12))
tick_locator = ticker.MaxNLocator(nbins=5)
cb1 = plt.colorbar(pc1, ax=avec[0])
cb1.ax.xaxis.set_label_position('top')
cb1.ax.set_xlabel(necbarstr,fontsize=fscb)
cb1.locator = tick_locator
cb1.update_ticks()
plot1 = avec[0].plot(rout,zout,'w.',markersize=3)
plt.tight_layout()
if wtimes:
plt.subplots_adjust(top=0.9)
spti = fig.suptitle('Parameters at {0} seconds'.format(int(itime[0]-t0)),fontsize=24)
return (fig,avec)
def lagdict2ionocont(DataLags, NoiseLags, sensdict, simparams, time_vec):
"""This function will take the data and noise lags and create an instance of the
Ionocontanier class. This function will also apply the summation rule to the lags.
Inputs
DataLags - A dictionary """
# Pull in Location Data
angles = simparams['angles']
ang_data = sp.array([[iout[0], iout[1]] for iout in angles])
rng_vec = simparams['Rangegates']
# pull in other data
pulsewidth = len(simparams['Pulse']) * sensdict['t_s']
txpower = sensdict['Pt']
if sensdict['Name'].lower() in ['risr', 'pfisr', 'risr-n']:
Ksysvec = sensdict['Ksys']
else:
beamlistlist = sp.array(simparams['outangles']).astype(int)
inplist = sp.array([i[0] for i in beamlistlist])
Ksysvec = sensdict['Ksys'][inplist]
ang_data_temp = ang_data.copy()
ang_data = sp.array(
[ang_data_temp[i].mean(axis=0) for i in beamlistlist])
sumrule = simparams['SUMRULE']
rng_vec2 = simparams['Rangegatesfinal']
minrg = -sumrule[0].min()
maxrg = len(rng_vec) - sumrule[1].max()
Nrng2 = len(rng_vec2)
# Copy the lags
lagsData = DataLags['ACF'].copy()
# Set up the constants for the lags so they are now
# in terms of density fluxtuations.
angtile = sp.tile(ang_data, (Nrng2, 1))
rng_rep = sp.repeat(rng_vec2, ang_data.shape[0], axis=0)
coordlist = sp.zeros((len(rng_rep), 3))
[coordlist[:, 0], coordlist[:, 1:]] = [rng_rep, angtile]
(Nt, Nbeams, Nrng, Nlags) = lagsData.shape
rng3d = sp.tile(rng_vec[sp.newaxis, sp.newaxis, :, sp.newaxis],
(Nt, Nbeams, 1, Nlags)) * 1e3
ksys3d = sp.tile(Ksysvec[sp.newaxis, :, sp.newaxis, sp.newaxis],
(Nt, 1, Nrng, Nlags))
radar2acfmult = rng3d * rng3d / (pulsewidth * txpower * ksys3d)
pulses = sp.tile(DataLags['Pulses'][:, :, sp.newaxis, sp.newaxis],
(1, 1, Nrng, Nlags))
time_vec = time_vec[:Nt]
# Divid lags by number of pulses
lagsData = lagsData / pulses
# Set up the noise lags and divid out the noise.
lagsNoise = NoiseLags['ACF'].copy()
lagsNoise = sp.mean(lagsNoise, axis=2)
pulsesnoise = sp.tile(NoiseLags['Pulses'][:, :, sp.newaxis], (1, 1, Nlags))
lagsNoise = lagsNoise / pulsesnoise
lagsNoise = sp.tile(lagsNoise[:, :, sp.newaxis, :], (1, 1, Nrng, 1))
# subtract out noise lags
lagsData = lagsData - lagsNoise
# Calculate a variance using equation 2 from Hysell's 2008 paper. Done use full covariance matrix because assuming nearly diagonal.
# multiply the data and the sigma by inverse of the scaling from the radar
lagsData = lagsData * radar2acfmult
lagsNoise = lagsNoise * radar2acfmult
# Apply summation rule
# lags transposed from (time,beams,range,lag)to (range,lag,time,beams)
lagsData = sp.transpose(lagsData, axes=(2, 3, 0, 1))
lagsNoise = sp.transpose(lagsNoise, axes=(2, 3, 0, 1))
lagsDatasum = sp.zeros((Nrng2, Nlags, Nt, Nbeams), dtype=lagsData.dtype)
lagsNoisesum = sp.zeros((Nrng2, Nlags, Nt, Nbeams), dtype=lagsNoise.dtype)
for irngnew, irng in enumerate(sp.arange(minrg, maxrg)):
for ilag in range(Nlags):
lagsDatasum[irngnew,
ilag] = lagsData[irng + sumrule[0, ilag]:irng +
sumrule[1, ilag] + 1,
ilag].sum(axis=0)
lagsNoisesum[irngnew,
ilag] = lagsNoise[irng + sumrule[0, ilag]:irng +
sumrule[1, ilag] + 1,
ilag].sum(axis=0)
# Put everything in a parameter list
Paramdata = sp.zeros((Nbeams * Nrng2, Nt, Nlags), dtype=lagsData.dtype)
# Put everything in a parameter list
# transpose from (range,lag,time,beams) to (beams,range,time,lag)
lagsDatasum = sp.transpose(lagsDatasum, axes=(3, 0, 2, 1))
lagsNoisesum = sp.transpose(lagsNoisesum, axes=(3, 0, 2, 1))
# Get the covariance matrix
pulses_s = sp.transpose(pulses, axes=(1, 2, 0, 3))[:, :Nrng2]
Cttout = makeCovmat(lagsDatasum, lagsNoisesum, pulses_s, Nlags)
Paramdatasig = sp.zeros((Nbeams * Nrng2, Nt, Nlags, Nlags),
dtype=Cttout.dtype)
curloc = 0
for irng in range(Nrng2):
for ibeam in range(Nbeams):
Paramdata[curloc] = lagsDatasum[ibeam, irng].copy()
Paramdatasig[curloc] = Cttout[ibeam, irng].copy()
curloc += 1
ionodata = IonoContainer(coordlist,
Paramdata,
times=time_vec,
ver=1,
paramnames=sp.arange(Nlags) * sensdict['t_s'])
ionosigs = IonoContainer(
coordlist,
Paramdatasig,
times=time_vec,
ver=1,
paramnames=sp.arange(Nlags * Nlags).reshape(Nlags, Nlags) *
sensdict['t_s'])
return (ionodata, ionosigs)
