def makespectrums(basedir,configfile,remakealldata):
""" This will make all of the spectra for a set of data and save it in a
folder in basedir called Spectrums. It is assumed that the data in the Origparams
is time tagged in the title with a string seperated by a white space and the
rest of the file name. For example ~/DATA/Basedir/0 origdata.h5.
Inputs:
basedir: A string for the directory that will hold all of the data for the simulation.
configfile: The configuration file for the simulation.
"""
dirio=('Origparams','Spectrums')
inputdir = os.path.join(basedir,dirio[0])
outputdir = os.path.join(basedir,dirio[1])
# determine the list of h5 files in Origparams directory
dirlist = glob.glob(os.path.join(inputdir,'*.h5'))
# Make the lists of numbers and file names for the dictionary
(listorder,timevector,filenumbering,timebeg,time_s) = IonoContainer.gettimes(dirlist)
slist = [dirlist[ikey] for ikey in listorder]
(sensdict,simparams) = readconfigfile(configfile)
# Delete data
outfiles = glob.glob(os.path.join(outputdir,'*.h5'))
for ifile in outfiles:
os.remove(ifile)
for inum, curfile in zip(timebeg,slist):
outfile = os.path.join(outputdir,str(inum)+' spectrum.h5')
print('Processing file {0} starting at {1}\n'.format(os.path.split(curfile)[1]
,datetime.now()))
curiono = IonoContainer.readh5(curfile)
curiono.makespectruminstanceopen(specfuncs.ISRSspecmake,sensdict,
simparams['numpoints']).saveh5(outfile)
print('Finished file {0} starting at {1}\n'.format(os.path.split(curfile)[1],datetime.now()))
def applymat(basedir,configfile,optinputs):
""" This function apply the matrix version of the space time ambiugty function
to the ACFs and save the outcome in h5 files within the directory ACFMat.
Inputs:
basedir: A string for the directory that will hold all of the data for the simulation.
configfile: The configuration file for the simulation.
"""
dirio = ('Spectrums','Mat','ACFMat')
inputdir = os.path.join(basedir,dirio[0])
outputdir2 = os.path.join(basedir,dirio[2])
dirlist = glob.glob(os.path.join(inputdir,'*.h5'))
(listorder,timevector,filenumbering,timebeg,time_s) = IonoContainer.gettimes(dirlist)
Ionolist = [dirlist[ikey] for ikey in listorder]
RSTO = RadarSpaceTimeOperator(Ionolist,configfile,timevector,mattype='matrix')
Ionoout = RSTO.mult_iono(Ionolist)
outfile=os.path.join(outputdir2,'00lags.h5')
Ionoout.saveh5(outfile)
def makeradardata(basedir,configfile,remakealldata):
""" This function will make the radar data and create the acf estimates.
Inputs:
basedir: A string for the directory that will hold all of the data for the simulation.
configfile: The configuration file for the simulation.
remakealldata: A bool that determines if the radar data is remade. If false
only the acfs will be estimated using the radar that is already made."""
dirio = ('Spectrums','Radardata','ACF')
inputdir = os.path.join(basedir,dirio[0])
outputdir = os.path.join(basedir,dirio[1])
outputdir2 = os.path.join(basedir,dirio[2])
# determine the list of h5 files in Origparams directory
dirlist = glob.glob(os.path.join(inputdir,'*.h5'))
# Make the lists of numbers and file names for the dictionary
if len(dirlist)>0:
(listorder,timevector,filenumbering,timebeg,time_s) = IonoContainer.gettimes(dirlist)
Ionodict = {timebeg[itn]:dirlist[it] for itn, it in enumerate(listorder)}
else:
Ionodict = {0:os.path.join(inputdir,'00.h5')}
# Find all of the raw data files
radardatalist = glob.glob(os.path.join(outputdir,'*RawData.h5'))
if radardatalist and (not remakealldata):
# XXX need to work on time stuff
outlist2 = radardatalist
else:
outlist2 = None
# create the radar data file class
rdata = RadarDataFile(Ionodict,configfile,outputdir,outfilelist=outlist2)
# From the ACFs and uncertainties
(ionoout,ionosig) = rdata.processdataiono()
# save the acfs and uncertianties in ionocontainer h5 files.
ionoout.saveh5(os.path.join(outputdir2,'00lags.h5'))
ionosig.saveh5(os.path.join(outputdir2,'00sigs.h5'))
return ()
def makehistdata(params, maindir):
""" This will make the histogram data for the statistics.
Inputs
params - A list of parameters that will have statistics created
maindir - The directory that the simulation data is held.
Outputs
datadict - A dictionary with the data values in numpy arrays. The keys are param names.
errordict - A dictionary with the data values in numpy arrays. The keys are param names.
errdictrel - A dictionary with the error values in numpy arrays, normalized by the correct value. The keys are param names.
"""
ffit = os.path.join(maindir, 'Fitted', 'fitteddata.h5')
inputfiledir = os.path.join(maindir, 'Origparams')
paramslower = [ip.lower() for ip in params]
eparamslower = ['n' + ip.lower() for ip in params]
# set up data dictionary
errordict = {ip: [] for ip in params}
errordictrel = {ip: [] for ip in params}
#Read in fitted data
Ionofit = IonoContainer.readh5(ffit)
times = Ionofit.Time_Vector
dataloc = Ionofit.Sphere_Coords
pnames = Ionofit.Param_Names
pnameslower = sp.array([ip.lower() for ip in pnames.flatten()])
p2fit = [
sp.argwhere(ip == pnameslower)[0][0] if ip in pnameslower else None
for ip in paramslower
]
datadict = {
ip: Ionofit.Param_List[:, :, p2fit[ipn]].flatten()
for ipn, ip in enumerate(params)
}
ep2fit = [
sp.argwhere(ip == pnameslower)[0][0] if ip in pnameslower else None
for ip in eparamslower
]
edatadict = {
ip: Ionofit.Param_List[:, :, ep2fit[ipn]].flatten()
for ipn, ip in enumerate(params)
}
# Determine which input files are to be used.
dirlist = glob.glob(os.path.join(inputfiledir, '*.h5'))
sortlist, outime, filelisting, timebeg, timelist_s = IonoContainer.gettimes(
dirlist)
time2files = []
for itn, itime in enumerate(times):
log1 = (outime[:, 0] >= itime[0]) & (outime[:, 0] < itime[1])
log2 = (outime[:, 1] > itime[0]) & (outime[:, 1] <= itime[1])
log3 = (outime[:, 0] <= itime[0]) & (outime[:, 1] > itime[1])
tempindx = sp.where(log1 | log2 | log3)[0]
time2files.append(filelisting[tempindx])
curfilenum = -1
for iparam, pname in enumerate(params):
curparm = paramslower[iparam]
# Use Ne from input to compare the ne derived from the power.
if curparm == 'nepow':
curparm = 'ne'
datalist = []
for itn, itime in enumerate(times):
for iplot, filenum in enumerate(time2files[itn]):
filenum = int(filenum)
if curfilenum != filenum:
curfilenum = filenum
datafilename = dirlist[filenum]
Ionoin = IonoContainer.readh5(datafilename)
if ('ti' in paramslower) or ('vi' in paramslower):
Ionoin = maketi(Ionoin)
pnames = Ionoin.Param_Names
pnameslowerin = sp.array(
[ip.lower() for ip in pnames.flatten()])
prmloc = sp.argwhere(curparm == pnameslowerin)
if prmloc.size != 0:
curprm = prmloc[0][0]
# build up parameter vector bs the range values by finding the closest point in space in the input
curdata = sp.zeros(len(dataloc))
for irngn, curcoord in enumerate(dataloc):
tempin = Ionoin.getclosestsphere(curcoord, [itime])[0]
Ntloc = tempin.shape[0]
tempin = sp.reshape(tempin, (Ntloc, len(pnameslowerin)))
curdata[irngn] = tempin[0, curprm]
datalist.append(curdata)
errordict[pname] = datadict[pname] - sp.hstack(datalist)
errordictrel[pname] = 100. * errordict[pname] / sp.absolute(
sp.hstack(datalist))
return datadict, errordict, errordictrel, edatadict
def plotbeamparametersv2(times,
configfile,
maindir,
fitdir='Fitted',
params=['Ne'],
filetemplate='params',
suptitle='Parameter Comparison',
werrors=False,
nelog=True):
""" This function will plot the desired parameters for each beam along range.
The values of the input and measured parameters will be plotted
Inputs
Times - A list of times that will be plotted.
configfile - The INI file with the simulation parameters that will be useds.
maindir - The directory the images will be saved in.
params - List of Parameter names that will be ploted. These need to match
in the ionocontainer names.
filetemplate - The first part of a the file names.
suptitle - The supertitle for the plots.
werrors - A bools that determines if the errors will be plotted.
"""
sns.set_style("whitegrid")
sns.set_context("notebook")
# rc('text', usetex=True)
ffit = os.path.join(maindir, fitdir, 'fitteddata.h5')
inputfiledir = os.path.join(maindir, 'Origparams')
(sensdict, simparams) = readconfigfile(configfile)
paramslower = [ip.lower() for ip in params]
Nt = len(times)
Np = len(params)
#Read in fitted data
Ionofit = IonoContainer.readh5(ffit)
dataloc = Ionofit.Sphere_Coords
pnames = Ionofit.Param_Names
pnameslower = sp.array([ip.lower() for ip in pnames.flatten()])
p2fit = [
sp.argwhere(ip == pnameslower)[0][0] if ip in pnameslower else None
for ip in paramslower
]
time2fit = [None] * Nt
for itn, itime in enumerate(times):
filear = sp.argwhere(Ionofit.Time_Vector >= itime)
if len(filear) == 0:
filenum = len(Ionofit.Time_Vector) - 1
else:
filenum = filear[0][0]
time2fit[itn] = filenum
times_int = [Ionofit.Time_Vector[i] for i in time2fit]
# determine the beams
angles = dataloc[:, 1:]
rng = sp.unique(dataloc[:, 0])
b = np.ascontiguousarray(angles).view(
np.dtype((np.void, angles.dtype.itemsize * angles.shape[1])))
_, idx, invidx = np.unique(b, return_index=True, return_inverse=True)
beamlist = angles[idx]
Nb = beamlist.shape[0]
# Determine which imput files are to be used.
dirlist = glob.glob(os.path.join(inputfiledir, '*.h5'))
filesonly = [
os.path.splitext(os.path.split(ifile)[-1])[0] for ifile in dirlist
]
sortlist, outime, outfilelist, timebeg, timelist_s = IonoContainer.gettimes(
dirlist)
timelist = sp.array([int(i.split()[0]) for i in filesonly])
time2file = [None] * Nt
time2intime = [None] * Nt
# go through times find files and then times in files
for itn, itime in enumerate(times):
filear = sp.argwhere(timelist >= itime)
if len(filear) == 0:
filenum = [len(timelist) - 1]
else:
filenum = filear[0]
flist1 = []
timeinflist = []
for ifile in filenum:
filetimes = timelist_s[ifile]
log1 = (filetimes[:, 0] >= times_int[itn][0]) & (filetimes[:, 0] <
times_int[itn][1])
log2 = (filetimes[:, 1] > times_int[itn][0]) & (filetimes[:, 1] <=
times_int[itn][1])
log3 = (filetimes[:, 0] <= times_int[itn][0]) & (filetimes[:, 1] >
times_int[itn][1])
log4 = (filetimes[:, 0] > times_int[itn][0]) & (filetimes[:, 1] <
times_int[itn][1])
curtimes1 = sp.where(log1 | log2 | log3 | log4)[0].tolist()
flist1 = flist1 + [ifile] * len(curtimes1)
timeinflist = timeinflist + curtimes1
time2intime[itn] = timeinflist
time2file[itn] = flist1
nfig = int(sp.ceil(Nt * Nb * Np / 9.0))
imcount = 0
curfilenum = -1
# Loop for the figures
for i_fig in range(nfig):
lines = [None] * 2
labels = [None] * 2
(figmplf, axmat) = plt.subplots(3, 3, figsize=(20, 15), facecolor='w')
axvec = axmat.flatten()
# loop that goes through each axis loops through each parameter, beam
# then time.
for iax, ax in enumerate(axvec):
if imcount >= Nt * Nb * Np:
break
itime = int(sp.floor(imcount / Nb / Np))
iparam = int(imcount / Nb - Np * itime)
ibeam = int(imcount - (itime * Np * Nb + iparam * Nb))
curbeam = beamlist[ibeam]
altlist = sp.sin(curbeam[1] * sp.pi / 180.) * rng
curparm = paramslower[iparam]
# Use Ne from input to compare the ne derived from the power.
if curparm == 'nepow':
curparm_in = 'ne'
else:
curparm_in = curparm
curcoord = sp.zeros(3)
curcoord[1:] = curbeam
for iplot, filenum in enumerate(time2file[itime]):
if curfilenum != filenum:
curfilenum = filenum
datafilename = dirlist[filenum]
Ionoin = IonoContainer.readh5(datafilename)
if ('ti' in paramslower) or ('vi' in paramslower):
Ionoin = maketi(Ionoin)
pnames = Ionoin.Param_Names
pnameslowerin = sp.array(
[ip.lower() for ip in pnames.flatten()])
prmloc = sp.argwhere(curparm_in == pnameslowerin)
if prmloc.size != 0:
curprm = prmloc[0][0]
# build up parameter vector bs the range values by finding the closest point in space in the input
curdata = sp.zeros(len(rng))
for irngn, irng in enumerate(rng):
curcoord[0] = irng
tempin = Ionoin.getclosestsphere(curcoord)[0][
time2intime[itime]]
Ntloc = tempin.shape[0]
tempin = sp.reshape(tempin, (Ntloc, len(pnameslowerin)))
curdata[irngn] = tempin[0, curprm]
#actual plotting of the input data
lines[0] = ax.plot(curdata,
altlist,
marker='o',
c='b',
linewidth=2)[0]
labels[0] = 'Input Parameters'
# Plot fitted data for the axis
indxkep = np.argwhere(invidx == ibeam)[:, 0]
curfit = Ionofit.Param_List[indxkep, time2fit[itime],
p2fit[iparam]]
rng_fit = dataloc[indxkep, 0]
alt_fit = rng_fit * sp.sin(curbeam[1] * sp.pi / 180.)
errorexist = 'n' + paramslower[iparam] in pnameslower
if errorexist and werrors:
eparam = sp.argwhere('n' +
paramslower[iparam] == pnameslower)[0][0]
curerror = Ionofit.Param_List[indxkep, time2fit[itime], eparam]
lines[1] = ax.errorbar(curfit,
alt_fit,
xerr=curerror,
fmt='-.',
c='g',
linewidth=2)[0]
else:
lines[1] = ax.plot(curfit,
alt_fit,
marker='o',
c='g',
linewidth=2)[0]
labels[1] = 'Fitted Parameters'
# get and plot the input data
numplots = len(time2file[itime])
# set the limit for the parameter
if curparm_in != 'ne':
ax.set(xlim=[0.75 * sp.amin(curfit), 1.25 * sp.amax(curfit)])
if curparm == 'vi':
ax.set(xlim=[
-1.25 * sp.amax(sp.absolute(curfit)), 1.25 *
sp.amax(sp.absolute(curfit))
])
if (curparm_in == 'ne') and nelog:
ax.set_xscale('log')
ax.set_xlabel(params[iparam])
ax.set_ylabel('Alt km')
ax.set_title(
'{0} vs Altitude, Time: {1}s Az: {2}$^o$ El: {3}$^o$'.format(
params[iparam], times[itime], *curbeam))
imcount = imcount + 1
# save figure
figmplf.suptitle(suptitle, fontsize=20)
if None in labels:
labels.remove(None)
lines.remove(None)
plt.figlegend(lines,
labels,
loc='lower center',
ncol=5,
labelspacing=0.)
fname = filetemplate + '_{0:0>3}.png'.format(i_fig)
plt.savefig(fname)
plt.close(figmplf)
