def makespectrums(basedir,configfile,remakealldata): """ This will make all of the spectra for a set of data and save it in a folder in basedir called Spectrums. It is assumed that the data in the Origparams is time tagged in the title with a string seperated by a white space and the rest of the file name. For example ~/DATA/Basedir/0 origdata.h5. Inputs: basedir: A string for the directory that will hold all of the data for the simulation. configfile: The configuration file for the simulation. """ dirio=('Origparams','Spectrums') inputdir = os.path.join(basedir,dirio[0]) outputdir = os.path.join(basedir,dirio[1]) # determine the list of h5 files in Origparams directory dirlist = glob.glob(os.path.join(inputdir,'*.h5')) # Make the lists of numbers and file names for the dictionary (listorder,timevector,filenumbering,timebeg,time_s) = IonoContainer.gettimes(dirlist) slist = [dirlist[ikey] for ikey in listorder] (sensdict,simparams) = readconfigfile(configfile) # Delete data outfiles = glob.glob(os.path.join(outputdir,'*.h5')) for ifile in outfiles: os.remove(ifile) for inum, curfile in zip(timebeg,slist): outfile = os.path.join(outputdir,str(inum)+' spectrum.h5') print('Processing file {0} starting at {1}\n'.format(os.path.split(curfile)[1] ,datetime.now())) curiono = IonoContainer.readh5(curfile) curiono.makespectruminstanceopen(specfuncs.ISRSspecmake,sensdict, simparams['numpoints']).saveh5(outfile) print('Finished file {0} starting at {1}\n'.format(os.path.split(curfile)[1],datetime.now()))
def applymat(basedir,configfile,optinputs): """ This function apply the matrix version of the space time ambiugty function to the ACFs and save the outcome in h5 files within the directory ACFMat. Inputs: basedir: A string for the directory that will hold all of the data for the simulation. configfile: The configuration file for the simulation. """ dirio = ('Spectrums','Mat','ACFMat') inputdir = os.path.join(basedir,dirio[0]) outputdir2 = os.path.join(basedir,dirio[2]) dirlist = glob.glob(os.path.join(inputdir,'*.h5')) (listorder,timevector,filenumbering,timebeg,time_s) = IonoContainer.gettimes(dirlist) Ionolist = [dirlist[ikey] for ikey in listorder] RSTO = RadarSpaceTimeOperator(Ionolist,configfile,timevector,mattype='matrix') Ionoout = RSTO.mult_iono(Ionolist) outfile=os.path.join(outputdir2,'00lags.h5') Ionoout.saveh5(outfile)
def makeradardata(basedir,configfile,remakealldata): """ This function will make the radar data and create the acf estimates. Inputs: basedir: A string for the directory that will hold all of the data for the simulation. configfile: The configuration file for the simulation. remakealldata: A bool that determines if the radar data is remade. If false only the acfs will be estimated using the radar that is already made.""" dirio = ('Spectrums','Radardata','ACF') inputdir = os.path.join(basedir,dirio[0]) outputdir = os.path.join(basedir,dirio[1]) outputdir2 = os.path.join(basedir,dirio[2]) # determine the list of h5 files in Origparams directory dirlist = glob.glob(os.path.join(inputdir,'*.h5')) # Make the lists of numbers and file names for the dictionary if len(dirlist)>0: (listorder,timevector,filenumbering,timebeg,time_s) = IonoContainer.gettimes(dirlist) Ionodict = {timebeg[itn]:dirlist[it] for itn, it in enumerate(listorder)} else: Ionodict = {0:os.path.join(inputdir,'00.h5')} # Find all of the raw data files radardatalist = glob.glob(os.path.join(outputdir,'*RawData.h5')) if radardatalist and (not remakealldata): # XXX need to work on time stuff outlist2 = radardatalist else: outlist2 = None # create the radar data file class rdata = RadarDataFile(Ionodict,configfile,outputdir,outfilelist=outlist2) # From the ACFs and uncertainties (ionoout,ionosig) = rdata.processdataiono() # save the acfs and uncertianties in ionocontainer h5 files. ionoout.saveh5(os.path.join(outputdir2,'00lags.h5')) ionosig.saveh5(os.path.join(outputdir2,'00sigs.h5')) return ()
def makehistdata(params, maindir): """ This will make the histogram data for the statistics. Inputs params - A list of parameters that will have statistics created maindir - The directory that the simulation data is held. Outputs datadict - A dictionary with the data values in numpy arrays. The keys are param names. errordict - A dictionary with the data values in numpy arrays. The keys are param names. errdictrel - A dictionary with the error values in numpy arrays, normalized by the correct value. The keys are param names. """ ffit = os.path.join(maindir, 'Fitted', 'fitteddata.h5') inputfiledir = os.path.join(maindir, 'Origparams') paramslower = [ip.lower() for ip in params] eparamslower = ['n' + ip.lower() for ip in params] # set up data dictionary errordict = {ip: [] for ip in params} errordictrel = {ip: [] for ip in params} #Read in fitted data Ionofit = IonoContainer.readh5(ffit) times = Ionofit.Time_Vector dataloc = Ionofit.Sphere_Coords pnames = Ionofit.Param_Names pnameslower = sp.array([ip.lower() for ip in pnames.flatten()]) p2fit = [ sp.argwhere(ip == pnameslower)[0][0] if ip in pnameslower else None for ip in paramslower ] datadict = { ip: Ionofit.Param_List[:, :, p2fit[ipn]].flatten() for ipn, ip in enumerate(params) } ep2fit = [ sp.argwhere(ip == pnameslower)[0][0] if ip in pnameslower else None for ip in eparamslower ] edatadict = { ip: Ionofit.Param_List[:, :, ep2fit[ipn]].flatten() for ipn, ip in enumerate(params) } # Determine which input files are to be used. dirlist = glob.glob(os.path.join(inputfiledir, '*.h5')) sortlist, outime, filelisting, timebeg, timelist_s = IonoContainer.gettimes( dirlist) time2files = [] for itn, itime in enumerate(times): log1 = (outime[:, 0] >= itime[0]) & (outime[:, 0] < itime[1]) log2 = (outime[:, 1] > itime[0]) & (outime[:, 1] <= itime[1]) log3 = (outime[:, 0] <= itime[0]) & (outime[:, 1] > itime[1]) tempindx = sp.where(log1 | log2 | log3)[0] time2files.append(filelisting[tempindx]) curfilenum = -1 for iparam, pname in enumerate(params): curparm = paramslower[iparam] # Use Ne from input to compare the ne derived from the power. if curparm == 'nepow': curparm = 'ne' datalist = [] for itn, itime in enumerate(times): for iplot, filenum in enumerate(time2files[itn]): filenum = int(filenum) if curfilenum != filenum: curfilenum = filenum datafilename = dirlist[filenum] Ionoin = IonoContainer.readh5(datafilename) if ('ti' in paramslower) or ('vi' in paramslower): Ionoin = maketi(Ionoin) pnames = Ionoin.Param_Names pnameslowerin = sp.array( [ip.lower() for ip in pnames.flatten()]) prmloc = sp.argwhere(curparm == pnameslowerin) if prmloc.size != 0: curprm = prmloc[0][0] # build up parameter vector bs the range values by finding the closest point in space in the input curdata = sp.zeros(len(dataloc)) for irngn, curcoord in enumerate(dataloc): tempin = Ionoin.getclosestsphere(curcoord, [itime])[0] Ntloc = tempin.shape[0] tempin = sp.reshape(tempin, (Ntloc, len(pnameslowerin))) curdata[irngn] = tempin[0, curprm] datalist.append(curdata) errordict[pname] = datadict[pname] - sp.hstack(datalist) errordictrel[pname] = 100. * errordict[pname] / sp.absolute( sp.hstack(datalist)) return datadict, errordict, errordictrel, edatadict
def plotbeamparametersv2(times, configfile, maindir, fitdir='Fitted', params=['Ne'], filetemplate='params', suptitle='Parameter Comparison', werrors=False, nelog=True): """ This function will plot the desired parameters for each beam along range. The values of the input and measured parameters will be plotted Inputs Times - A list of times that will be plotted. configfile - The INI file with the simulation parameters that will be useds. maindir - The directory the images will be saved in. params - List of Parameter names that will be ploted. These need to match in the ionocontainer names. filetemplate - The first part of a the file names. suptitle - The supertitle for the plots. werrors - A bools that determines if the errors will be plotted. """ sns.set_style("whitegrid") sns.set_context("notebook") # rc('text', usetex=True) ffit = os.path.join(maindir, fitdir, 'fitteddata.h5') inputfiledir = os.path.join(maindir, 'Origparams') (sensdict, simparams) = readconfigfile(configfile) paramslower = [ip.lower() for ip in params] Nt = len(times) Np = len(params) #Read in fitted data Ionofit = IonoContainer.readh5(ffit) dataloc = Ionofit.Sphere_Coords pnames = Ionofit.Param_Names pnameslower = sp.array([ip.lower() for ip in pnames.flatten()]) p2fit = [ sp.argwhere(ip == pnameslower)[0][0] if ip in pnameslower else None for ip in paramslower ] time2fit = [None] * Nt for itn, itime in enumerate(times): filear = sp.argwhere(Ionofit.Time_Vector >= itime) if len(filear) == 0: filenum = len(Ionofit.Time_Vector) - 1 else: filenum = filear[0][0] time2fit[itn] = filenum times_int = [Ionofit.Time_Vector[i] for i in time2fit] # determine the beams angles = dataloc[:, 1:] rng = sp.unique(dataloc[:, 0]) b = np.ascontiguousarray(angles).view( np.dtype((np.void, angles.dtype.itemsize * angles.shape[1]))) _, idx, invidx = np.unique(b, return_index=True, return_inverse=True) beamlist = angles[idx] Nb = beamlist.shape[0] # Determine which imput files are to be used. dirlist = glob.glob(os.path.join(inputfiledir, '*.h5')) filesonly = [ os.path.splitext(os.path.split(ifile)[-1])[0] for ifile in dirlist ] sortlist, outime, outfilelist, timebeg, timelist_s = IonoContainer.gettimes( dirlist) timelist = sp.array([int(i.split()[0]) for i in filesonly]) time2file = [None] * Nt time2intime = [None] * Nt # go through times find files and then times in files for itn, itime in enumerate(times): filear = sp.argwhere(timelist >= itime) if len(filear) == 0: filenum = [len(timelist) - 1] else: filenum = filear[0] flist1 = [] timeinflist = [] for ifile in filenum: filetimes = timelist_s[ifile] log1 = (filetimes[:, 0] >= times_int[itn][0]) & (filetimes[:, 0] < times_int[itn][1]) log2 = (filetimes[:, 1] > times_int[itn][0]) & (filetimes[:, 1] <= times_int[itn][1]) log3 = (filetimes[:, 0] <= times_int[itn][0]) & (filetimes[:, 1] > times_int[itn][1]) log4 = (filetimes[:, 0] > times_int[itn][0]) & (filetimes[:, 1] < times_int[itn][1]) curtimes1 = sp.where(log1 | log2 | log3 | log4)[0].tolist() flist1 = flist1 + [ifile] * len(curtimes1) timeinflist = timeinflist + curtimes1 time2intime[itn] = timeinflist time2file[itn] = flist1 nfig = int(sp.ceil(Nt * Nb * Np / 9.0)) imcount = 0 curfilenum = -1 # Loop for the figures for i_fig in range(nfig): lines = [None] * 2 labels = [None] * 2 (figmplf, axmat) = plt.subplots(3, 3, figsize=(20, 15), facecolor='w') axvec = axmat.flatten() # loop that goes through each axis loops through each parameter, beam # then time. for iax, ax in enumerate(axvec): if imcount >= Nt * Nb * Np: break itime = int(sp.floor(imcount / Nb / Np)) iparam = int(imcount / Nb - Np * itime) ibeam = int(imcount - (itime * Np * Nb + iparam * Nb)) curbeam = beamlist[ibeam] altlist = sp.sin(curbeam[1] * sp.pi / 180.) * rng curparm = paramslower[iparam] # Use Ne from input to compare the ne derived from the power. if curparm == 'nepow': curparm_in = 'ne' else: curparm_in = curparm curcoord = sp.zeros(3) curcoord[1:] = curbeam for iplot, filenum in enumerate(time2file[itime]): if curfilenum != filenum: curfilenum = filenum datafilename = dirlist[filenum] Ionoin = IonoContainer.readh5(datafilename) if ('ti' in paramslower) or ('vi' in paramslower): Ionoin = maketi(Ionoin) pnames = Ionoin.Param_Names pnameslowerin = sp.array( [ip.lower() for ip in pnames.flatten()]) prmloc = sp.argwhere(curparm_in == pnameslowerin) if prmloc.size != 0: curprm = prmloc[0][0] # build up parameter vector bs the range values by finding the closest point in space in the input curdata = sp.zeros(len(rng)) for irngn, irng in enumerate(rng): curcoord[0] = irng tempin = Ionoin.getclosestsphere(curcoord)[0][ time2intime[itime]] Ntloc = tempin.shape[0] tempin = sp.reshape(tempin, (Ntloc, len(pnameslowerin))) curdata[irngn] = tempin[0, curprm] #actual plotting of the input data lines[0] = ax.plot(curdata, altlist, marker='o', c='b', linewidth=2)[0] labels[0] = 'Input Parameters' # Plot fitted data for the axis indxkep = np.argwhere(invidx == ibeam)[:, 0] curfit = Ionofit.Param_List[indxkep, time2fit[itime], p2fit[iparam]] rng_fit = dataloc[indxkep, 0] alt_fit = rng_fit * sp.sin(curbeam[1] * sp.pi / 180.) errorexist = 'n' + paramslower[iparam] in pnameslower if errorexist and werrors: eparam = sp.argwhere('n' + paramslower[iparam] == pnameslower)[0][0] curerror = Ionofit.Param_List[indxkep, time2fit[itime], eparam] lines[1] = ax.errorbar(curfit, alt_fit, xerr=curerror, fmt='-.', c='g', linewidth=2)[0] else: lines[1] = ax.plot(curfit, alt_fit, marker='o', c='g', linewidth=2)[0] labels[1] = 'Fitted Parameters' # get and plot the input data numplots = len(time2file[itime]) # set the limit for the parameter if curparm_in != 'ne': ax.set(xlim=[0.75 * sp.amin(curfit), 1.25 * sp.amax(curfit)]) if curparm == 'vi': ax.set(xlim=[ -1.25 * sp.amax(sp.absolute(curfit)), 1.25 * sp.amax(sp.absolute(curfit)) ]) if (curparm_in == 'ne') and nelog: ax.set_xscale('log') ax.set_xlabel(params[iparam]) ax.set_ylabel('Alt km') ax.set_title( '{0} vs Altitude, Time: {1}s Az: {2}$^o$ El: {3}$^o$'.format( params[iparam], times[itime], *curbeam)) imcount = imcount + 1 # save figure figmplf.suptitle(suptitle, fontsize=20) if None in labels: labels.remove(None) lines.remove(None) plt.figlegend(lines, labels, loc='lower center', ncol=5, labelspacing=0.) fname = filetemplate + '_{0:0>3}.png'.format(i_fig) plt.savefig(fname) plt.close(figmplf)