def runinversion(basedir,configfile,acfdir='ACF',invtype='tik'):
""" """
costdir = os.path.join(basedir,'Cost')
pname=os.path.join(costdir,'cost{0}-{1}.pickle'.format(acfdir,invtype))
pickleFile = open(pname, 'rb')
alpha_arr=pickle.load(pickleFile)[-1]
pickleFile.close()
ionoinfname=os.path.join(basedir,acfdir,'00lags.h5')
ionoin=IonoContainer.readh5(ionoinfname)
dirio = ('Spectrums','Mat','ACFMat')
inputdir = os.path.join(basedir,dirio[0])
dirlist = glob.glob(os.path.join(inputdir,'*.h5'))
(listorder,timevector,filenumbering,timebeg,time_s) = IonoContainer.gettimes(dirlist)
Ionolist = [dirlist[ikey] for ikey in listorder]
if acfdir.lower()=='acf':
ionosigname=os.path.join(basedir,acfdir,'00sigs.h5')
ionosigin=IonoContainer.readh5(ionosigname)
nl,nt,np1,np2=ionosigin.Param_List.shape
sigs=ionosigin.Param_List.reshape((nl*nt,np1,np2))
sigsmean=sp.nanmean(sigs,axis=0)
sigdiag=sp.diag(sigsmean)
sigsout=sp.power(sigdiag/sigdiag[0],.5).real
alpha_arr=sp.ones_like(alpha_arr)*alpha_arr[0]
acfloc='ACFInv'
elif acfdir.lower()=='acfmat':
mattype='matrix'
acfloc='ACFMatInv'
mattype='sim'
RSTO = RadarSpaceTimeOperator(Ionolist,configfile,timevector,mattype=mattype)
if 'perryplane' in basedir.lower() or 'SimpData':
rbounds=[-500,500]
else:
rbounds=[0,500]
ionoout=invertRSTO(RSTO,ionoin,alpha_list=alpha_arr,invtype=invtype,rbounds=rbounds)[0]
outfile=os.path.join(basedir,acfloc,'00lags{0}.h5'.format(invtype))
ionoout.saveh5(outfile)
if acfdir=='ACF':
lagsDatasum=ionoout.Param_List
# !!! This is done to speed up development
lagsNoisesum=sp.zeros_like(lagsDatasum)
Nlags=lagsDatasum.shape[-1]
pulses_s=RSTO.simparams['Tint']/RSTO.simparams['IPP']
Ctt=makeCovmat(lagsDatasum,lagsNoisesum,pulses_s,Nlags)
outfile=os.path.join(basedir,acfloc,'00sigs{0}.h5'.format(invtype))
ionoout.Param_List=Ctt
ionoout.Param_Names=sp.repeat(ionoout.Param_Names[:,sp.newaxis],Nlags,axis=1)
ionoout.saveh5(outfile)
def applymat(basedir, configfile, optinputs):
"""
This function apply the matrix version of the space time ambiugty function
to the ACFs and save the outcome in h5 files within the directory ACFMat.
Inputs:
basedir: A string for the directory that will hold all of the data for the simulation.
configfile: The configuration file for the simulation.
"""
dirio = ('Spectrums', 'Mat', 'ACFMat')
basedir = Path(basedir)
inputdir = basedir.joinpath(dirio[0])
outputdir2 = basedir.joinpath(dirio[2])
dirlist = [str(i) for i in inputdir.glob('*.h5')]
(listorder, timevector, _, _, _) = IonoContainer.gettimes(dirlist)
ionolist = [dirlist[ikey] for ikey in listorder]
rsto = RadarSpaceTimeOperator(ionolist, configfile, timevector, mattype='matrix')
ionoout = rsto.mult_iono(ionolist)
outfile = outputdir2.joinpath('00lags.h5')
ionoout.saveh5(str(outfile))
def makeradardata(basedir,configfile,remakealldata):
""" This function will make the radar data and create the acf estimates.
Inputs:
basedir: A string for the directory that will hold all of the data for the simulation.
configfile: The configuration file for the simulation.
remakealldata: A bool that determines if the radar data is remade. If false
only the acfs will be estimated using the radar that is already made."""
dirio = ('Spectrums', 'Radardata', 'ACF')
inputdir = basedir/dirio[0]
outputdir = basedir/dirio[1]
outputdir2 = basedir/dirio[2]
# determine the list of h5 files in Origparams directory
dirlist = [str(i) for i in inputdir.glob('*.h5')]
# Make the lists of numbers and file names for the dictionary
if len(dirlist) > 0:
(listorder, _, _, timebeg, _) = IonoContainer.gettimes(dirlist)
Ionodict = {timebeg[itn]:dirlist[it] for itn, it in enumerate(listorder)}
else:
Ionodict = {0.:str(inputdir.joinpath('00.h5'))}
# Find all of the raw data files
radardatalist = outputdir.glob('*RawData.h5')
if radardatalist and (not remakealldata):
# XXX need to work on time stuff
outlist2 = radardatalist
else:
outlist2 = None
# create the radar data file class
rdata = RadarDataFile(Ionodict, configfile, outputdir, outfilelist=outlist2)
# From the ACFs and uncertainties
(ionoout, ionosig) = rdata.processdataiono()
# save the acfs and uncertianties in ionocontainer h5 files.
ionoout.saveh5(str(outputdir2.joinpath('00lags.h5')))
ionosig.saveh5(str(outputdir2.joinpath('00sigs.h5')))
return ()
def makespectrums(basedir, configfile, printlines=True):
""" This will make all of the spectra for a set of data and save it in a
folder in basedir called Spectrums. It is assumed that the data in the Origparams
is time tagged in the title with a string seperated by a white space and the
rest of the file name. For example ~/DATA/Basedir/0 origdata.h5.
Inputs:
basedir: A string for the directory that will hold all of the data for the simulation.
configfile: The configuration file for the simulation.
"""
basedir = Path(basedir).expanduser()
dirio = ('Origparams', 'Spectrums')
inputdir = basedir/dirio[0]
outputdir = basedir/dirio[1]
# determine the list of h5 files in Origparams directory
dirlist = sorted(inputdir.glob('*.h5'))
# Make the lists of numbers and file names for the dictionary
(listorder, _, _, timebeg, _) = IonoContainer.gettimes(dirlist)
slist = [dirlist[ikey] for ikey in listorder]
(sensdict, simparams) = readconfigfile(configfile)
# Delete data
outfiles = outputdir.glob('*.h5')
for ifile in outfiles:
ifile.unlink()
for inum, curfile in zip(timebeg, slist):
outfile = outputdir / (str(inum)+' spectrum.h5')
update_progress(float(inum)/float(len(slist)),
'Processing file {} starting at {}'.format(curfile.name, datetime.now()))
curiono = IonoContainer.readh5(str(curfile))
curiono.makespectruminstanceopen(specfuncs.ISRSspecmake, sensdict,
int(simparams['numpoints']), float(inum),
float(len(slist)), printlines).saveh5(str(outfile))
update_progress(float(inum+1)/float(len(slist)),
'Finished file {} starting at {}'.format(curfile.name, datetime.now()))
def makehistdata(params, maindir):
""" This will make the histogram data for the statistics.
Inputs
params - A list of parameters that will have statistics created
maindir - The directory that the simulation data is held.
Outputs
datadict - A dictionary with the data values in numpy arrays. The keys are param names.
errordict - A dictionary with the data values in numpy arrays. The keys are param names.
errdictrel - A dictionary with the error values in numpy arrays, normalized by the correct value. The keys are param names.
"""
maindir = Path(maindir)
ffit = maindir.joinpath('Fitted', 'fitteddata.h5')
inputfiledir = maindir.joinpath('Origparams')
paramslower = [ip.lower() for ip in params]
eparamslower = ['n' + ip.lower() for ip in params]
# set up data dictionary
errordict = {ip: [] for ip in params}
errordictrel = {ip: [] for ip in params}
#Read in fitted data
Ionofit = IonoContainer.readh5(str(ffit))
times = Ionofit.Time_Vector
dataloc = Ionofit.Sphere_Coords
rng = dataloc[:, 0]
rng_log = sp.logical_and(rng > 200., rng < 400)
dataloc_out = dataloc[rng_log]
pnames = Ionofit.Param_Names
pnameslower = sp.array([ip.lower() for ip in pnames.flatten()])
p2fit = [
sp.argwhere(ip == pnameslower)[0][0] if ip in pnameslower else None
for ip in paramslower
]
datadict = {
ip: Ionofit.Param_List[rng_log, :, p2fit[ipn]].flatten()
for ipn, ip in enumerate(params)
}
ep2fit = [
sp.argwhere(ip == pnameslower)[0][0] if ip in pnameslower else None
for ip in eparamslower
]
edatadict = {
ip: Ionofit.Param_List[rng_log, :, ep2fit[ipn]].flatten()
for ipn, ip in enumerate(params)
}
# Determine which input files are to be used.
dirlist = [str(i) for i in inputfiledir.glob('*.h5')]
_, outime, filelisting, _, _ = IonoContainer.gettimes(dirlist)
time2files = []
for itn, itime in enumerate(times):
log1 = (outime[:, 0] >= itime[0]) & (outime[:, 0] < itime[1])
log2 = (outime[:, 1] > itime[0]) & (outime[:, 1] <= itime[1])
log3 = (outime[:, 0] <= itime[0]) & (outime[:, 1] > itime[1])
tempindx = sp.where(log1 | log2 | log3)[0]
time2files.append(filelisting[tempindx])
curfilenum = -1
for iparam, pname in enumerate(params):
curparm = paramslower[iparam]
# Use Ne from input to compare the ne derived from the power.
if curparm == 'nepow':
curparm = 'ne'
datalist = []
for itn, itime in enumerate(times):
for filenum in time2files[itn]:
filenum = int(filenum)
if curfilenum != filenum:
curfilenum = filenum
datafilename = dirlist[filenum]
Ionoin = IonoContainer.readh5(datafilename)
if ('ti' in paramslower) or ('vi' in paramslower):
Ionoin = maketi(Ionoin)
pnames = Ionoin.Param_Names
pnameslowerin = sp.array(
[ip.lower() for ip in pnames.flatten()])
prmloc = sp.argwhere(curparm == pnameslowerin)
if prmloc.size != 0:
curprm = prmloc[0][0]
# build up parameter vector bs the range values by finding the closest point in space in the input
curdata = sp.zeros(len(dataloc_out))
for irngn, curcoord in enumerate(dataloc_out):
tempin = Ionoin.getclosestsphere(curcoord, [itime])[0]
Ntloc = tempin.shape[0]
tempin = sp.reshape(tempin, (Ntloc, len(pnameslowerin)))
curdata[irngn] = tempin[0, curprm]
datalist.append(curdata)
errordict[pname] = datadict[pname] - sp.hstack(datalist)
errordictrel[pname] = 100. * errordict[pname] / sp.absolute(
sp.hstack(datalist))
return datadict, errordict, errordictrel, edatadict
def parametersweep(basedir,configfile,acfdir='ACF',invtype='tik'):
"""
This function will run the inversion numerious times with different constraint
parameters. This will create a directory called cost and place.
Input
basedir - The directory that holds all of the data for the simulator.
configfile - The ini file for the simulation.
acfdir - The directory within basedir that hold the acfs to be inverted.
invtype - The inversion method that will be tested. Can be tik, tikd, and tv.
"""
alpha_sweep=sp.logspace(-3.5,sp.log10(7),25)
costdir = os.path.join(basedir,'Cost')
ionoinfname=os.path.join(basedir,acfdir,'00lags.h5')
ionoin=IonoContainer.readh5(ionoinfname)
dirio = ('Spectrums','Mat','ACFMat')
inputdir = os.path.join(basedir,dirio[0])
dirlist = glob.glob(os.path.join(inputdir,'*.h5'))
(listorder,timevector,filenumbering,timebeg,time_s) = IonoContainer.gettimes(dirlist)
Ionolist = [dirlist[ikey] for ikey in listorder]
RSTO = RadarSpaceTimeOperator(Ionolist,configfile,timevector,mattype='Sim')
npts=RSTO.simparams['numpoints']
ionospec=makeionocombined(dirlist)
if npts==ionospec.Param_List.shape[-1]:
tau,acfin=spect2acf(ionospec.Param_Names,ionospec.Param_List)
nloc,ntimes=acfin.shape[:2]
ambmat=RSTO.simparams['amb_dict']['WttMatrix']
np=ambmat.shape[0]
acfin_amb=sp.zeros((nloc,ntimes,np),dtype=acfin.dtype)
# get the original acf
ambmat=RSTO.simparams['amb_dict']['WttMatrix']
np=ambmat.shape[0]
for iloc,locarr in enumerate(acfin):
for itime,acfarr in enumerate(locarr):
acfin_amb[iloc,itime]=sp.dot(ambmat,acfarr)
acfin_amb=acfin_amb[:,0]
else:
acfin_amb=ionospec.Param_List[:,0]
if not os.path.isdir(costdir):
os.mkdir(costdir)
# pickle file stuff
pname=os.path.join(costdir,'cost{0}-{1}.pickle'.format(acfdir,invtype))
alpha_list=[]
errorlist=[]
errorlaglist=[]
datadiflist=[]
constlist=[]
if 'perryplane' in basedir.lower() or 'SimpData':
rbounds=[-500,500]
else:
rbounds=[0,500]
alpha_list_new=alpha_sweep.tolist()
for i in alpha_list:
if i in alpha_list_new:
alpha_list_new.remove(i)
for i in alpha_list_new:
ionoout,datadif,constdif=invertRSTO(RSTO,ionoin,alpha_list=i,invtype=invtype,rbounds=rbounds,Nlin=1)
datadiflist.append(datadif)
constlist.append(constdif)
acfout=ionoout.Param_List[:,0]
alpha_list.append(i)
outdata=sp.power(sp.absolute(acfout-acfin_amb),2)
aveerror=sp.sqrt(sp.nanmean(outdata,axis=0))
errorlaglist.append(aveerror)
errorlist.append(sp.nansum(aveerror))
pickleFile = open(pname, 'wb')
pickle.dump([alpha_list,errorlist,datadiflist,constlist,errorlaglist],pickleFile)
pickleFile.close()
mkalphalist(pname)
alphaarr=sp.array(alpha_list)
errorarr=sp.array(errorlist)
errorlagarr=sp.array(errorlaglist)
datadif=sp.array(datadiflist)
constdif=sp.array(constlist)
fig,axlist,axmain=plotalphaerror(alphaarr,errorarr,errorlagarr)
fig.savefig(os.path.join(costdir,'cost{0}-{1}.png'.format(acfdir,invtype)))
fig,axlist=plotLcurve(alphaarr,datadif,constdif)
fig.savefig(os.path.join(costdir,'lcurve{0}-{1}.png'.format(acfdir,invtype)))
def makehistdata(params,maindir):
""" This will make the histogram data for the statistics.
Inputs
params - A list of parameters that will have statistics created
maindir - The directory that the simulation data is held.
Outputs
datadict - A dictionary with the data values in numpy arrays. The keys are param names.
errordict - A dictionary with the data values in numpy arrays. The keys are param names.
errdictrel - A dictionary with the error values in numpy arrays, normalized by the correct value. The keys are param names.
"""
maindir=Path(maindir)
ffit = maindir.joinpath('Fitted','fitteddata.h5')
inputfiledir = maindir.joinpath('Origparams')
paramslower = [ip.lower() for ip in params]
eparamslower = ['n'+ip.lower() for ip in params]
# set up data dictionary
errordict = {ip:[] for ip in params}
errordictrel = {ip:[] for ip in params}
#Read in fitted data
Ionofit = IonoContainer.readh5(ffit)
times=Ionofit.Time_Vector
dataloc = Ionofit.Sphere_Coords
pnames = Ionofit.Param_Names
pnameslower = sp.array([ip.lower() for ip in pnames.flatten()])
p2fit = [sp.argwhere(ip==pnameslower)[0][0] if ip in pnameslower else None for ip in paramslower]
datadict = {ip:Ionofit.Param_List[:,:,p2fit[ipn]].flatten() for ipn, ip in enumerate(params)}
ep2fit = [sp.argwhere(ip==pnameslower)[0][0] if ip in pnameslower else None for ip in eparamslower]
edatadict = {ip:Ionofit.Param_List[:,:,ep2fit[ipn]].flatten() for ipn, ip in enumerate(params)}
# Determine which input files are to be used.
dirlist = [str(i) for i in inputfiledir.glob('*.h5')]
sortlist,outime,filelisting,timebeg,timelist_s = IonoContainer.gettimes(dirlist)
time2files = []
for itn,itime in enumerate(times):
log1 = (outime[:,0]>=itime[0]) & (outime[:,0]<itime[1])
log2 = (outime[:,1]>itime[0]) & (outime[:,1]<=itime[1])
log3 = (outime[:,0]<=itime[0]) & (outime[:,1]>itime[1])
tempindx = sp.where(log1|log2|log3)[0]
time2files.append(filelisting[tempindx])
curfilenum=-1
for iparam,pname in enumerate(params):
curparm = paramslower[iparam]
# Use Ne from input to compare the ne derived from the power.
if curparm == 'nepow':
curparm = 'ne'
datalist = []
for itn,itime in enumerate(times):
for iplot,filenum in enumerate(time2files[itn]):
filenum = int(filenum)
if curfilenum!=filenum:
curfilenum=filenum
datafilename = dirlist[filenum]
Ionoin = IonoContainer.readh5(datafilename)
if ('ti' in paramslower) or ('vi' in paramslower):
Ionoin = maketi(Ionoin)
pnames = Ionoin.Param_Names
pnameslowerin = sp.array([ip.lower() for ip in pnames.flatten()])
prmloc = sp.argwhere(curparm==pnameslowerin)
if prmloc.size !=0:
curprm = prmloc[0][0]
# build up parameter vector bs the range values by finding the closest point in space in the input
curdata = sp.zeros(len(dataloc))
for irngn, curcoord in enumerate(dataloc):
tempin = Ionoin.getclosestsphere(curcoord,[itime])[0]
Ntloc = tempin.shape[0]
tempin = sp.reshape(tempin,(Ntloc,len(pnameslowerin)))
curdata[irngn] = tempin[0,curprm]
datalist.append(curdata)
errordict[pname] = datadict[pname]-sp.hstack(datalist)
errordictrel[pname] = 100.*errordict[pname]/sp.absolute(sp.hstack(datalist))
return datadict,errordict,errordictrel,edatadict
