def main():
""" Test function for the RadarData class."""
t1 = time.time()
curpath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
testpath = os.path.join(os.path.split(curpath)[0],'Testdata')
Ionoin=IonoContainer.readh5(os.path.join(testpath,'lags.h5'))
inifile = os.path.join(testpath,'PFISRExample.pickle')
fitterone = Fitterionoconainer(Ionoin,inifile)
(fitteddata,fittederror) = fitterone.fitdata(ISRSfitfunction,simpstart)
(Nloc,Ntimes,nparams)=fitteddata.shape
fittederronly = fittederror[:,:,range(nparams),range(nparams)]
paramlist = sp.concatenate((fitteddata,fittederronly),axis=2)
paramnames = []
species = fitterone.simparams['species']
for isp in species[:-1]:
paramnames.append('Ni_'+isp)
paramnames.append('Ti_'+isp)
paramnames = paramnames+['Ne','Te','Vi']
paramnamese = ['n'+ip for ip in paramnames]
paranamsf = sp.array(paramnames+paramnamese)
Ionoout=IonoContainer(Ionoin.Sphere_Coords,paramlist,Ionoin.Time_Vector,ver =1,
coordvecs = Ionoin.Coord_Vecs, paramnames=paranamsf,species=species)
Ionoout.saveh5(os.path.join(testpath,'fittedtestdata.h5'))
t2 = time.time()
print(t2-t1)
def startvalfunc(Ne_init, loc, time, inputs):
""" This is a method to determine the start values for the fitter.
Inputs
Ne_init - A nloc x nt numpy array of the initial estimate of electron density. Basically
the zeroth lag of the ACF.
loc - A nloc x 3 numpy array of cartisian coordinates.
time - A nt x 2 numpy array of times in seconds
exinputs - A list of extra inputs allowed for by the fitter class. It only
has one element and its the name of the ionocontainer file holding the
rest of the start parameters.
Outputs
xarray - This is a numpy arrya of starting values for the fitter parmaeters."""
if isinstance(inputs, str):
if os.path.splitext(inputs)[-1] == '.h5':
Ionoin = IonoContainer.readh5(inputs)
elif os.path.splitext(inputs)[-1] == '.mat':
Ionoin = IonoContainer.readmat(inputs)
elif os.path.splitext(inputs)[-1] == '':
Ionoin = IonoContainer.makeionocombined(inputs)
elif isinstance(inputs, list):
Ionoin = IonoContainer.makeionocombined(inputs)
else:
Ionoin = inputs
numel = sp.prod(Ionoin.Param_List.shape[-2:]) + 1
xarray = sp.zeros((loc.shape[0], len(time), numel))
for ilocn, iloc in enumerate(loc):
(datast, vel) = Ionoin.getclosest(iloc, time)[:2]
datast[:, -1, 0] = Ne_init[ilocn, :]
ionoden = datast[:, :-1, 0]
ionodensum = sp.repeat(sp.sum(ionoden, axis=-1)[:, sp.newaxis],
ionoden.shape[-1],
axis=-1)
ionoden = sp.repeat(Ne_init[ilocn, :, sp.newaxis],
ionoden.shape[-1],
axis=-1) * ionoden / ionodensum
datast[:, :-1, 0] = ionoden
xarray[ilocn, :, :-1] = sp.reshape(datast, (len(time), numel - 1))
locmag = sp.sqrt(sp.sum(iloc * iloc))
ilocmat = sp.repeat(iloc[sp.newaxis, :], len(time), axis=0)
xarray[ilocn, :, -1] = sp.sum(vel * ilocmat) / locmag
return xarray
def main():
"""Testing function"""
t1 = time.time()
curpath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
testpath = os.path.join(os.path.split(curpath)[0],'Testdata')
inifile = os.path.join(testpath,'PFISRExample.pickle')
(sensdict,simparams) = readconfigfile(inifile)
testh5 = os.path.join(testpath,'testiono.h5')
ioncont = IonoContainer.readh5(testh5)
outfile = os.path.join(testpath,'testionospec.h5')
ioncont.makespectruminstanceopen(specfunctions.ISRSspecmake,sensdict,
simparams['numpoints']).saveh5(outfile)
radardata = RadarDataFile({0.0:outfile},inifile,testpath)
ionoout = radardata.processdataiono()
ionoout.saveh5(os.path.join(testpath,'lags.h5'))
t2 = time.time()
print(t2-t1)
def __init__(self,Ionodict,inifile, outdir,outfilelist=None):
"""This function will create an instance of the RadarData class. It will
take in the values and create the class and make raw IQ data.
Inputs:
sensdict - A dictionary of sensor parameters
angles - A list of tuples which the first position is the az angle
and the second position is the el angle.
IPP - The interpulse period in seconds represented as a float.
Tint - The integration time in seconds as a float. This will be the
integration time of all of the beams.
time_lim - The length of time of the simulation the number of time points
will be calculated.
pulse - A numpy array that represents the pulse shape.
rng_lims - A numpy array of length 2 that holds the min and max range
that the radar will cover."""
(sensdict,simparams) = readconfigfile(inifile)
self.simparams = simparams
N_angles = len(self.simparams['angles'])
NNs = self.simparams['NNs']
self.sensdict = sensdict
Npall = sp.floor(self.simparams['TimeLim']/self.simparams['IPP'])
Npall = sp.floor(Npall/N_angles)*N_angles
Np = Npall/N_angles
print "All spectrums created already"
filetimes = Ionodict.keys()
filetimes.sort()
ftimes = sp.array(filetimes)
simdtype = self.simparams['dtype']
pulsetimes = sp.arange(Npall)*self.simparams['IPP']
pulsefile = sp.array([sp.where(itimes-ftimes>=0)[0][-1] for itimes in pulsetimes])
# differentiate between phased arrays and dish antennas
if sensdict['Name'].lower() in ['risr','pfisr']:
beams = sp.tile(sp.arange(N_angles),Npall/N_angles)
else:
# for dish arrays
brate = simparams['beamrate']
beams2 = sp.repeat(sp.arange(N_angles),brate)
beam3 = sp.concatenate((beams2,beams2[:-1,::-1]))
beams = sp.tile(beam3,Npall/len(beam3))
pulsen = sp.repeat(sp.arange(Np),N_angles)
pt_list = []
pb_list = []
pn_list = []
fname_list = []
self.datadir = outdir
self.maindir = os.path.dirname(os.path.abspath(outdir))
self.procdir =os.path.join(self.maindir,'ACF')
self.procdir
if outfilelist is None:
print('\nData Now being created.')
Noisepwr = v_Boltz*sensdict['Tsys']*sensdict['BandWidth']
self.outfilelist = []
for ifn, ifilet in enumerate(filetimes):
outdict = {}
ifile = Ionodict[ifilet]
print('\tData from {0:d} of {1:d} being processed Name: {2:s}.'.format(ifn,len(filetimes),os.path.split(ifile)[1]))
curcontainer = IonoContainer.readh5(ifile)
pnts = pulsefile==ifn
pt =pulsetimes[pnts]
pb = beams[pnts]
pn = pulsen[pnts].astype(int)
(rawdata,outdict['SpecsUsed'])= self.__makeTime__(pt,curcontainer.Time_Vector,
curcontainer.Sphere_Coords, curcontainer.Param_List,pb)
Noise = sp.sqrt(Noisepwr/2)*(sp.random.randn(*rawdata.shape).astype(simdtype)+
1j*sp.random.randn(*rawdata.shape).astype(simdtype))
outdict['AddedNoise'] =Noise
outdict['RawData'] = rawdata+Noise
outdict['RawDatanonoise'] = rawdata
outdict['NoiseData'] = sp.sqrt(Noisepwr/2)*(sp.random.randn(len(pn),NNs).astype(simdtype)+
1j*sp.random.randn(len(pn),NNs).astype(simdtype))
outdict['Pulses']=pn
outdict['Beams']=pb
outdict['Time'] = pt
fname = '{0:d} RawData.h5'.format(ifn)
newfn = os.path.join(self.datadir,fname)
self.outfilelist.append(newfn)
dict2h5(newfn,outdict)
#Listing info
pt_list.append(pt)
pb_list.append(pb)
pn_list.append(pn)
fname_list.append(fname)
infodict = {'Files':fname_list,'Time':pt_list,'Beams':pb_list,'Pulses':pn_list}
dict2h5(os.path.join(outdir,'INFO.h5'),infodict)
else:
self.outfilelist=outfilelist
def mult_iono(self, ionoin_list):
"""
This will apply the forward model to the contents of an ionocontainer object. It is assuming that
this is an ionocontainer holding the spectra.
"""
ntout = self.Time_Out.shape[0]
nlout = self.Cart_Coords_Out.shape[0]
blist_in, blist_out = self.blocklocs
amb_dict = self.simparams['amb_dict']
ambmat = amb_dict['WttMatrix']
overlaps = self.overlaps
t_s = self.sensdict['t_s']
if isinstance(ionoin_list, list) or isinstance(ionoin_list, str):
Iono_in = makeionocombined(ionoin_list)
else:
Iono_in = ionoin_list
ionocart = Iono_in.Cart_Coords
if self.simparams['numpoints'] == Iono_in.Param_List.shape[-1]:
tau, acf = spect2acf(Iono_in.Param_Names, Iono_in.Param_List)
np = ambmat.shape[0]
else:
acf = Iono_in.Param_List
np = acf.shape[-1]
np_in = acf.shape[-1]
tau_out = t_s * sp.arange(np)
outdata = sp.zeros((nlout, ntout, np), dtype=acf.dtype)
assert sp.allclose(
ionocart,
self.Cart_Coords_In), "Spatial Coordinates need to be the same"
for it_out in range(ntout):
overlists = overlaps[it_out]
irows = blist_out[it_out]
curintimes = [i[0] for i in overlists]
curintratio = [i[1] for i in overlists]
if self.mattype.lower() == 'sim':
curintimes = [curintimes[0]]
curintratio = [1.]
cur_outmat = self.RSTMat[irows[0]:irows[1], :]
icols = blist_in[it_out]
cur_mat = cur_outmat[:, icols[0]:icols[1]]
for i_it, it_in in enumerate(curintimes):
tempdata = sp.zeros((np_in, nlout), dtype=acf.dtype)
for iparam in range(np_in):
tempdata[iparam] = cur_mat.dot(acf[:, it_in, iparam])
if self.simparams['numpoints'] == Iono_in.Param_List.shape[-1]:
tempdata = sp.dot(ambmat, tempdata)
outdata[:, it_out] = sp.transpose(
tempdata) * curintratio[i_it] + outdata[:, it_out]
outiono = IonoContainer(self.Sphere_Coords_Out,
outdata,
times=self.Time_Out,
sensor_loc=Iono_in.Sensor_loc,
ver=1,
coordvecs=['r', 'theta', 'phi'],
paramnames=tau_out)
return outiono
def __init__(self,Ionodict,inifile, outdir,outfilelist=None):
"""This function will create an instance of the RadarData class. It will
take in the values and create the class and make raw IQ data.
Inputs:
sensdict - A dictionary of sensor parameters
angles - A list of tuples which the first position is the az angle
and the second position is the el angle.
IPP - The interpulse period in seconds represented as a float.
Tint - The integration time in seconds as a float. This will be the
integration time of all of the beams.
time_lim - The length of time of the simulation the number of time points
will be calculated.
pulse - A numpy array that represents the pulse shape.
rng_lims - A numpy array of length 2 that holds the min and max range
that the radar will cover."""
(sensdict,simparams) = readconfigfile(inifile)
self.simparams = simparams
N_angles = len(self.simparams['angles'])
NNs = int(self.simparams['NNs'])
self.sensdict = sensdict
Npall = sp.floor(self.simparams['TimeLim']/self.simparams['IPP'])
Npall = sp.floor(Npall/N_angles)*N_angles
Np = Npall/N_angles
print "All spectrums created already"
filetimes = Ionodict.keys()
filetimes.sort()
ftimes = sp.array(filetimes)
simdtype = self.simparams['dtype']
pulsetimes = sp.arange(Npall)*self.simparams['IPP'] +ftimes.min()
pulsefile = sp.array([sp.where(itimes-ftimes>=0)[0][-1] for itimes in pulsetimes])
# differentiate between phased arrays and dish antennas
if sensdict['Name'].lower() in ['risr','pfisr','risr-n']:
beams = sp.tile(sp.arange(N_angles),Npall/N_angles)
else:
# for dish arrays
brate = simparams['beamrate']
beams2 = sp.repeat(sp.arange(N_angles),brate)
beam3 = sp.concatenate((beams2,beams2[::-1]))
ntile = sp.ceil(Npall/len(beam3))
leftover = Npall-ntile*len(beam3)
if ntile>0:
beams = sp.tile(beam3,ntile)
beams=sp.concatenate((beams,beam3[:leftover]))
else:
beams=beam3[:leftover]
pulsen = sp.repeat(sp.arange(Np),N_angles)
pt_list = []
pb_list = []
pn_list = []
fname_list = []
self.datadir = outdir
self.maindir = os.path.dirname(os.path.abspath(outdir))
self.procdir =os.path.join(self.maindir,'ACF')
if outfilelist is None:
print('\nData Now being created.')
Noisepwr = v_Boltz*sensdict['Tsys']*sensdict['BandWidth']
self.outfilelist = []
for ifn, ifilet in enumerate(filetimes):
outdict = {}
ifile = Ionodict[ifilet]
print('\tData from {0:d} of {1:d} being processed Name: {2:s}.'.format(ifn,len(filetimes),
os.path.split(ifile)[1]))
curcontainer = IonoContainer.readh5(ifile)
if ifn==0:
self.timeoffset=curcontainer.Time_Vector[0,0]
pnts = pulsefile==ifn
pt =pulsetimes[pnts]
pb = beams[pnts]
pn = pulsen[pnts].astype(int)
rawdata= self.__makeTime__(pt,curcontainer.Time_Vector,
curcontainer.Sphere_Coords, curcontainer.Param_List,pb)
Noise = sp.sqrt(Noisepwr/2)*(sp.random.randn(*rawdata.shape).astype(simdtype)+
1j*sp.random.randn(*rawdata.shape).astype(simdtype))
outdict['AddedNoise'] =Noise
outdict['RawData'] = rawdata+Noise
outdict['RawDatanonoise'] = rawdata
outdict['NoiseData'] = sp.sqrt(Noisepwr/2)*(sp.random.randn(len(pn),NNs).astype(simdtype)+
1j*sp.random.randn(len(pn),NNs).astype(simdtype))
outdict['Pulses']=pn
outdict['Beams']=pb
outdict['Time'] = pt
fname = '{0:d} RawData.h5'.format(ifn)
newfn = os.path.join(self.datadir,fname)
self.outfilelist.append(newfn)
dict2h5(newfn,outdict)
#Listing info
pt_list.append(pt)
pb_list.append(pb)
pn_list.append(pn)
fname_list.append(fname)
infodict = {'Files':fname_list,'Time':pt_list,'Beams':pb_list,'Pulses':pn_list}
dict2h5(os.path.join(outdir,'INFO.h5'),infodict)
else:
infodict= h52dict(os.path.join(outdir,'INFO.h5'))
alltime=sp.hstack(infodict['Time'])
self.timeoffset=alltime.min()
self.outfilelist=outfilelist
def lagdict2ionocont(DataLags,NoiseLags,sensdict,simparams,time_vec):
"""This function will take the data and noise lags and create an instance of the
Ionocontanier class. This function will also apply the summation rule to the lags.
Inputs
DataLags - A dictionary """
# Pull in Location Data
angles = simparams['angles']
ang_data = sp.array([[iout[0],iout[1]] for iout in angles])
rng_vec = simparams['Rangegates']
# pull in other data
pulsewidth = len(simparams['Pulse'])*sensdict['t_s']
txpower = sensdict['Pt']
if sensdict['Name'].lower() in ['risr','pfisr','risr-n']:
Ksysvec = sensdict['Ksys']
else:
beamlistlist = sp.array(simparams['outangles']).astype(int)
inplist = sp.array([i[0] for i in beamlistlist])
Ksysvec = sensdict['Ksys'][inplist]
ang_data_temp = ang_data.copy()
ang_data = sp.array([ang_data_temp[i].mean(axis=0) for i in beamlistlist ])
sumrule = simparams['SUMRULE']
rng_vec2 = simparams['Rangegatesfinal']
minrg = -sumrule[0].min()
maxrg = len(rng_vec)-sumrule[1].max()
Nrng2 = len(rng_vec2)
# Copy the lags
lagsData= DataLags['ACF'].copy()
# Set up the constants for the lags so they are now
# in terms of density fluxtuations.
angtile = sp.tile(ang_data,(Nrng2,1))
rng_rep = sp.repeat(rng_vec2,ang_data.shape[0],axis=0)
coordlist=sp.zeros((len(rng_rep),3))
[coordlist[:,0],coordlist[:,1:]] = [rng_rep,angtile]
(Nt,Nbeams,Nrng,Nlags) = lagsData.shape
rng3d = sp.tile(rng_vec[sp.newaxis,sp.newaxis,:,sp.newaxis],(Nt,Nbeams,1,Nlags)) *1e3
ksys3d = sp.tile(Ksysvec[sp.newaxis,:,sp.newaxis,sp.newaxis],(Nt,1,Nrng,Nlags))
radar2acfmult = rng3d*rng3d/(pulsewidth*txpower*ksys3d)
pulses = sp.tile(DataLags['Pulses'][:,:,sp.newaxis,sp.newaxis],(1,1,Nrng,Nlags))
time_vec = time_vec[:Nt]
# Divid lags by number of pulses
lagsData = lagsData/pulses
# Set up the noise lags and divid out the noise.
lagsNoise=NoiseLags['ACF'].copy()
lagsNoise = sp.mean(lagsNoise,axis=2)
pulsesnoise = sp.tile(NoiseLags['Pulses'][:,:,sp.newaxis],(1,1,Nlags))
lagsNoise = lagsNoise/pulsesnoise
lagsNoise = sp.tile(lagsNoise[:,:,sp.newaxis,:],(1,1,Nrng,1))
# subtract out noise lags
lagsData = lagsData-lagsNoise
# Calculate a variance using equation 2 from Hysell's 2008 paper. Done use full covariance matrix because assuming nearly diagonal.
# multiply the data and the sigma by inverse of the scaling from the radar
lagsData = lagsData*radar2acfmult
lagsNoise = lagsNoise*radar2acfmult
# Apply summation rule
# lags transposed from (time,beams,range,lag)to (range,lag,time,beams)
lagsData = sp.transpose(lagsData,axes=(2,3,0,1))
lagsNoise = sp.transpose(lagsNoise,axes=(2,3,0,1))
lagsDatasum = sp.zeros((Nrng2,Nlags,Nt,Nbeams),dtype=lagsData.dtype)
lagsNoisesum = sp.zeros((Nrng2,Nlags,Nt,Nbeams),dtype=lagsNoise.dtype)
for irngnew,irng in enumerate(sp.arange(minrg,maxrg)):
for ilag in range(Nlags):
lagsDatasum[irngnew,ilag] = lagsData[irng+sumrule[0,ilag]:irng+sumrule[1,ilag]+1,ilag].sum(axis=0)
lagsNoisesum[irngnew,ilag] = lagsNoise[irng+sumrule[0,ilag]:irng+sumrule[1,ilag]+1,ilag].sum(axis=0)
# Put everything in a parameter list
Paramdata = sp.zeros((Nbeams*Nrng2,Nt,Nlags),dtype=lagsData.dtype)
# Put everything in a parameter list
# transpose from (range,lag,time,beams) to (beams,range,time,lag)
lagsDatasum = sp.transpose(lagsDatasum,axes=(3,0,2,1))
lagsNoisesum = sp.transpose(lagsNoisesum,axes=(3,0,2,1))
# Get the covariance matrix
pulses_s=sp.transpose(pulses,axes=(1,2,0,3))[:,:Nrng2]
Cttout=makeCovmat(lagsDatasum,lagsNoisesum,pulses_s,Nlags)
# R=sp.transpose(lagsDatasum/sp.sqrt(2.*pulses_s),axes=(3,0,1,2))
# Rw=sp.transpose(lagsNoisesum/sp.sqrt(2.*pulses_s),axes=(3,0,1,2))
# l=sp.arange(Nlags)
# T1,T2=sp.meshgrid(l,l)
# R0=R[sp.zeros_like(T1)]
# Rw0=Rw[sp.zeros_like(T1)]
# Td=sp.absolute(T1-T2)
# Tl = T1>T2
# R12 =R[Td]
# R12[Tl]=sp.conjugate(R12[Tl])
# Rw12 =Rw[Td]
# Rw12[Tl]=sp.conjugate(Rw12[Tl])
# Ctt=R0*R12+R[T1]*sp.conjugate(R[T2])+Rw0*Rw12+Rw[T1]*sp.conjugate(Rw[T2])
# Cttout = sp.transpose(Ctt,(2,3,4,0,1))
Paramdatasig = sp.zeros((Nbeams*Nrng2,Nt,Nlags,Nlags),dtype=Cttout.dtype)
curloc = 0
for irng in range(Nrng2):
for ibeam in range(Nbeams):
Paramdata[curloc] = lagsDatasum[ibeam,irng].copy()
Paramdatasig[curloc] = Cttout[ibeam,irng].copy()
curloc+=1
ionodata = IonoContainer(coordlist,Paramdata,times = time_vec,ver =1, paramnames=sp.arange(Nlags)*sensdict['t_s'])
ionosigs = IonoContainer(coordlist,Paramdatasig,times = time_vec,ver =1, paramnames=sp.arange(Nlags*Nlags).reshape(Nlags,Nlags)*sensdict['t_s'])
return (ionodata,ionosigs)
