def printList(self) -> List[str]: """Class information as a list of strings Returns ------- out : List[str] List of strings with information """ textLst: List[str] = [] textLst.append("Time data path = {}".format(self.timePath)) textLst.append("Spectra data path = {}".format(self.specPath)) textLst.append("Statistics data path = {}".format(self.statPath)) textLst.append("Mask data path = {}".format(self.maskPath)) textLst.append("TransFunc data path = {}".format(self.transFuncPath)) textLst.append("Calibration data path = {}".format(self.calPath)) textLst.append("Images data path = {}".format(self.imagePath)) textLst.append("Reference time = {}".format( self.refTime.strftime("%Y-%m-%d %H:%M:%S"))) textLst.append("Project start time = {}".format( self.projStart.strftime("%Y-%m-%d %H:%M:%S.%f"))) textLst.append("Project stop time = {}".format( self.projEnd.strftime("%Y-%m-%d %H:%M:%S.%f"))) textLst.append("Project found {} sites:".format(self.getNumSites())) for site in self.sites: textLst.append("{}\t\tstart: {}\tend: {}".format( site, self.getSiteData(site).siteStart, self.getSiteData(site).siteEnd, )) textLst.append("Sampling frequencies found in project (Hz): {}".format( listToString(self.getSampleFreqs()))) return textLst
def printList(self) -> List[str]: """Class information as a list of strings Returns ------- out : list List of strings with information """ textLst = [] textLst.append("Default options") textLst.append("\tInput Chans = {}".format( listToString(self.getInChans()))) textLst.append("\tOutput Chans = {}".format( listToString(self.getOutChans()))) textLst.append("\tRemote Chans = {}".format( listToString(self.getRemoteChans()))) textLst.append("\tPowers = {}".format(listToString( self.getPSDChans()))) textLst.append("\tCoherence pairs = {}".format( listToString(self.getCohPairs()))) textLst.append("\tPartial coherence = {}".format( listToString(self.getPolDirs()))) if len(self.getEvalFreq()) == 0: textLst.append("Evaluation frequencies = {}") else: textLst.append("Evaluation frequencies = {}".format( arrayToString(self.getEvalFreq()))) return textLst
def write(self, maskData: MaskData) -> None: """Write the maskData out to datapath Mask data is saved as a numpy binary object Parameters ---------- maskData : MaskData MaskData object """ infoName, winName = self.getFileNames(maskData.maskName, maskData.sampleFreq) infoFile = open(infoName, "w") # first write out constraints infoFile.write("{:.9f}\n".format(maskData.sampleFreq)) infoFile.write("{}\n".format(maskData.numLevels)) for iL in range(0, maskData.numLevels): infoFile.write("{}\n".format(listToString(maskData.evalFreq[iL]))) infoFile.write("{}\n".format(", ".join(maskData.stats))) # now write out the data file # first get a sorted list of all the evaluations frequencies to loop through evalFreq = sorted(list(maskData.constraints.keys())) for eFreq in evalFreq: infoFile.write("Frequency = {:.9f}\n".format(eFreq)) for stat in maskData.stats: infoFile.write("Statistic = {}\n".format(stat)) for component in maskData.constraints[eFreq][stat]: minVal = maskData.constraints[eFreq][stat][component][0] maxVal = maskData.constraints[eFreq][stat][component][1] infoFile.write("{}\t{}\t{}\t{}\n".format( component, minVal, maxVal, maskData.insideOut[eFreq][stat][component], )) # then loop through each infoFile.close() maskSize = 0 for eFreq in evalFreq: if len(maskData.maskWindows[eFreq]) > maskSize: maskSize = len(maskData.maskWindows[eFreq]) # create window mask array and initalise to -1 winMaskArray = np.ones(shape=(len(evalFreq), maskSize), dtype=int) * -1 # now fill the array for eIdx, eFreq in enumerate(evalFreq): lst = list(maskData.maskWindows[eFreq]) winMaskArray[eIdx, 0:len(lst)] = lst np.save(winName, winMaskArray)
def printListSection(self, section: str) -> List[str]: """Configuration section information as a list of strings Returns ------- out : List[str] List of strings with information """ textLst: List[str] = [] textLst.append("{:s}:".format(section)) for key, value in self.configParams[section].items(): textLst.append("\t{:s} = {}".format(key, value)) defaultOptions = "No defaults used" if len(self.configParams[section].defaults) > 0: defaultOptions = listToString(self.configParams[section].defaults) textLst.append("\tDefaulted options = {:s}".format(defaultOptions)) return textLst
def test_listToString() -> None: from resistics.common.print import listToString testlist = [1, 2, 4, 6, "mixed"] assert listToString(testlist) == "1, 2, 4, 6, mixed"
def calculateSpectra(projData: ProjectData, **kwargs) -> None: """Calculate spectra for the project time data The philosophy is that spectra are calculated out for all data and later limited using statistics and time constraints Parameters ---------- projData : ProjectData A project data object sites : str, List[str], optional Either a single site or a list of sites sampleFreqs : int, float, List[float], optional The frequencies in Hz for which to calculate the spectra. Either a single frequency or a list of them. chans : List[str], optional The channels for which to calculate out the spectra polreverse : Dict[str, bool] Keys are channels and values are boolean flags for reversing scale : Dict[str, float] Keys are channels and values are floats to multiply the channel data by calibrate : bool, optional Flag whether to calibrate the data or not notch : List[float], optional List of frequencies to notch filter : Dict, optional Filter parameters specdir : str, optional The spectra directory to save the spectra data in ncores : int, optional The number of cores to run the transfer function calculations on """ from resistics.spectra.io import SpectrumWriter from resistics.decimate.decimator import Decimator from resistics.window.windower import Windower from resistics.project.shortcuts import ( getCalibrator, getDecimationParameters, getWindowParameters, ) from resistics.project.preprocess import ( applyPolarisationReversalOptions, applyScaleOptions, applyCalibrationOptions, applyFilterOptions, applyNotchOptions, ) options = {} options["sites"] = projData.getSites() options["sampleFreqs"]: List[float] = projData.getSampleFreqs() options["chans"]: List[str] = [] options["polreverse"]: Union[bool, Dict[str, bool]] = False options["scale"]: Union[bool, Dict[str, float]] = False options["calibrate"]: bool = True options["notch"]: List[float] = [] options["filter"]: Dict = {} options["specdir"]: str = projData.config.configParams["Spectra"][ "specdir"] options["ncores"] = projData.config.getSpectraCores() options = parseKeywords(options, kwargs) # prepare calibrator cal = getCalibrator(projData.calPath, projData.config) if options["calibrate"]: cal.printInfo() datetimeRef = projData.refTime for site in options["sites"]: siteData = projData.getSiteData(site) siteData.printInfo() # calculate spectra for each frequency for sampleFreq in options["sampleFreqs"]: measurements = siteData.getMeasurements(sampleFreq) projectText( "Site {} has {:d} measurement(s) at sampling frequency {:.2f}". format(site, len(measurements), sampleFreq)) if len(measurements) == 0: continue # no data files at this sample rate for meas in measurements: projectText( "Calculating spectra for site {} and measurement {}". format(site, meas)) # get measurement start and end times - this is the time of the first and last sample reader = siteData.getMeasurement(meas) startTime = siteData.getMeasurementStart(meas) stopTime = siteData.getMeasurementEnd(meas) dataChans = (options["chans"] if len(options["chans"]) > 0 else reader.getChannels()) timeData = reader.getPhysicalData(startTime, stopTime, chans=dataChans) timeData.addComment(breakComment()) timeData.addComment("Calculating project spectra") timeData.addComment(projData.config.getConfigComment()) # apply various options applyPolarisationReversalOptions(options, timeData) applyScaleOptions(options, timeData) applyCalibrationOptions(options, cal, timeData, reader) applyFilterOptions(options, timeData) applyNotchOptions(options, timeData) # define decimation and window parameters decParams = getDecimationParameters(sampleFreq, projData.config) numLevels = decParams.numLevels winParams = getWindowParameters(decParams, projData.config) dec = Decimator(timeData, decParams) timeData.addComment( "Decimating with {} levels and {} frequencies per level". format(numLevels, decParams.freqPerLevel)) # loop through decimation levels for declevel in range(0, numLevels): # get the data for the current level check = dec.incrementLevel() if not check: break # not enough data timeData = dec.timeData # create the windower and give it window parameters for current level sampleFreqDec = dec.sampleFreq win = Windower( datetimeRef, timeData, winParams.getWindowSize(declevel), winParams.getOverlap(declevel), ) if win.numWindows < 2: break # do no more decimation # print information and add some comments projectText( "Calculating spectra for decimation level {}".format( declevel)) timeData.addComment( "Evaluation frequencies for this level {}".format( listToString( decParams.getEvalFrequenciesForLevel( declevel)))) timeData.addComment( "Windowing with window size {} samples and overlap {} samples" .format( winParams.getWindowSize(declevel), winParams.getOverlap(declevel), )) if projData.config.configParams["Spectra"]["applywindow"]: timeData.addComment( "Performing fourier transform with window function {}" .format(projData.config.configParams["Spectra"] ["windowfunc"])) else: timeData.addComment( "Performing fourier transform with no window function" ) # collect time data timeDataList = [] for iW in range(0, win.numWindows): timeDataList.append(win.getData(iW)) # open spectra file for saving specPath = os.path.join( siteData.getMeasurementSpecPath(meas), options["specdir"]) specWrite = SpectrumWriter(specPath, datetimeRef) specWrite.openBinaryForWriting( "spectra", declevel, sampleFreqDec, winParams.getWindowSize(declevel), winParams.getOverlap(declevel), win.winOffset, win.numWindows, dataChans, ) if options["ncores"] > 0: specDataList = multiSpectra( options["ncores"], timeDataList, sampleFreqDec, winParams.getWindowSize(declevel), projData.config.configParams, ) else: specDataList = calculateWindowSpectra( timeDataList, sampleFreqDec, winParams.getWindowSize(declevel), projData.config.configParams, ) # write out to spectra file for iW in range(0, win.numWindows): specWrite.writeBinary(specDataList[iW]) specWrite.writeCommentsFile(timeData.getComments()) specWrite.closeFile()
def calculateRemoteStatistics(projData: ProjectData, remoteSite: str, **kwargs): """Calculate statistics involving a remote reference site Parameters ---------- projData : ProjectData A project data instance remoteSite : str The name of the site to use as the remote site sites : List[str], optional A list of sites to calculate statistics for sampleFreqs : List[float], optional List of sampling frequencies for which to calculate statistics specdir : str, optional The spectra directory for which to calculate statistics remotestats : List[str], optional The statistics to calculate out. Acceptable statistics are: "RR_coherence", "RR_coherenceEqn", "RR_absvalEqn", "RR_transferFunction", "RR_resPhase". Configuration file values are used by default. """ from resistics.statistics.io import StatisticIO from resistics.statistics.calculator import StatisticCalculator from resistics.project.shortcuts import ( getDecimationParameters, getWindowParameters, getWindowSelector, ) options = {} options["sites"] = projData.getSites() options["sampleFreqs"] = projData.getSampleFreqs() options["chans"] = [] options["specdir"] = projData.config.configParams["Spectra"]["specdir"] options["remotestats"] = projData.config.configParams["Statistics"]["remotestats"] options["ncores"] = projData.config.getStatisticCores() options = parseKeywords(options, kwargs) projectText( "Calculating stats: {} for sites: {} with remote site {}".format( listToString(options["remotestats"]), listToString(options["sites"]), remoteSite, ) ) statIO = StatisticIO() for site in options["sites"]: siteData = projData.getSiteData(site) measurements = siteData.getMeasurements() for meas in measurements: sampleFreq = siteData.getMeasurementSampleFreq(meas) if sampleFreq not in options["sampleFreqs"]: continue projectText( "Calculating stats for site {}, measurement {} with reference {}".format( site, meas, remoteSite ) ) # decimation and window parameters decParams = getDecimationParameters(sampleFreq, projData.config) numLevels = decParams.numLevels winParams = getWindowParameters(decParams, projData.config) # create the window selector and find the shared windows winSelector = getWindowSelector(projData, decParams, winParams) winSelector.setSites([site, remoteSite]) winSelector.calcSharedWindows() # create the spectrum reader specReader = SpectrumReader( os.path.join(siteData.getMeasurementSpecPath(meas), options["specdir"]) ) # calculate statistics for decimation level if spectra file exists for declevel in range(0, numLevels): check = specReader.openBinaryForReading("spectra", declevel) if not check: continue # information regarding only this spectra file refTime = specReader.getReferenceTime() winSize = specReader.getWindowSize() winOlap = specReader.getWindowOverlap() numWindows = specReader.getNumWindows() evalFreq = decParams.getEvalFrequenciesForLevel(declevel) sampleFreqDec = specReader.getSampleFreq() globalOffset = specReader.getGlobalOffset() # find size of the intersection between the windows in this spectra file and the shared windows sharedWindows = winSelector.getSharedWindowsLevel(declevel) sharedWindowsMeas = sharedWindows.intersection( set(np.arange(globalOffset, globalOffset + numWindows)) ) sharedWindowsMeas = sorted(list(sharedWindowsMeas)) numSharedWindows = len(sharedWindowsMeas) statData = {} # create the statistic handlers for stat in options["remotestats"]: statElements = getStatElements(stat) statData[stat] = StatisticData( stat, refTime, sampleFreqDec, winSize, winOlap ) # with remote reference the number of windows is number of shared windows statData[stat].setStatParams( numSharedWindows, statElements, evalFreq ) statData[stat].comments = specReader.getComments() statData[stat].addComment(projData.config.getConfigComment()) statData[stat].addComment( "Calculating remote statistic: {}".format(stat) ) statData[stat].addComment( "Statistic components: {}".format(listToString(statElements)) ) # collect the spectra data spectraData, _globalIndices = specReader.readBinaryBatchGlobal( sharedWindowsMeas ) remoteData = [] for globalWindow in sharedWindowsMeas: _, remoteReader = winSelector.getSpecReaderForWindow( remoteSite, declevel, globalWindow ) remoteData.append(remoteReader.readBinaryWindowGlobal(globalWindow)) # calculate if options["ncores"] > 0: out = multiStatistics( options["ncores"], spectraData, evalFreq, options["remotestats"], remoteData=remoteData, ) for iW, globalWindow in enumerate(sharedWindowsMeas): for stat in options["remotestats"]: statData[stat].addStat(iW, globalWindow, out[iW][stat]) else: statCalculator = StatisticCalculator() for iW, globalWindow in enumerate(sharedWindowsMeas): winStatData = calculateWindowStatistics( spectraData[iW], evalFreq, options["remotestats"], remoteSpecData=remoteData[iW], statCalculator=statCalculator, ) for stat in options["remotestats"]: statData[stat].addStat(iW, globalWindow, winStatData[stat]) # save statistic for stat in options["remotestats"]: statIO.setDatapath( os.path.join( siteData.getMeasurementStatPath(meas), options["specdir"] ) ) statIO.write(statData[stat], declevel)
def calculateStatistics(projData: ProjectData, **kwargs): """Calculate statistics for sites Parameters ---------- projData : ProjectData A project data instance sites : List[str], optional A list of sites to calculate statistics for sampleFreqs : List[float], optional List of sampling frequencies for which to calculate statistics specdir : str, optional The spectra directory for which to calculate statistics stats : List[str], optional The statistics to calculate out. Acceptable values are: "absvalEqn" "coherence", "psd", "poldir", "transFunc", "resPhase", "partialcoh". Configuration file values are used by default. ncores : int, optional The number of cores to run the transfer function calculations on """ from resistics.statistics.io import StatisticIO from resistics.project.shortcuts import getDecimationParameters options = {} options["sites"] = projData.getSites() options["sampleFreqs"] = projData.getSampleFreqs() options["chans"] = [] options["specdir"] = projData.config.configParams["Spectra"]["specdir"] options["stats"] = projData.config.configParams["Statistics"]["stats"] options["ncores"] = projData.config.getStatisticCores() options = parseKeywords(options, kwargs) projectText( "Calculating stats: {} for sites: {}".format( listToString(options["stats"]), listToString(options["sites"]) ) ) # loop through sites and calculate statistics statIO = StatisticIO() for site in options["sites"]: siteData = projData.getSiteData(site) measurements = siteData.getMeasurements() for meas in measurements: sampleFreq = siteData.getMeasurementSampleFreq(meas) if sampleFreq not in options["sampleFreqs"]: continue projectText( "Calculating stats for site {}, measurement {}".format(site, meas) ) decParams = getDecimationParameters(sampleFreq, projData.config) numLevels = decParams.numLevels specReader = SpectrumReader( os.path.join(siteData.getMeasurementSpecPath(meas), options["specdir"]) ) # calculate statistics for decimation level if spectra file exists for declevel in range(0, numLevels): check = specReader.openBinaryForReading("spectra", declevel) if not check: continue refTime = specReader.getReferenceTime() winSize = specReader.getWindowSize() winOlap = specReader.getWindowOverlap() numWindows = specReader.getNumWindows() sampleFreqDec = specReader.getSampleFreq() evalFreq = decParams.getEvalFrequenciesForLevel(declevel) # dictionary for saving statistic data statData = {} for stat in options["stats"]: statElements = getStatElements(stat) statData[stat] = StatisticData( stat, refTime, sampleFreqDec, winSize, winOlap ) statData[stat].setStatParams(numWindows, statElements, evalFreq) statData[stat].comments = specReader.getComments() statData[stat].addComment(projData.config.getConfigComment()) statData[stat].addComment("Calculating statistic: {}".format(stat)) statData[stat].addComment( "Statistic components: {}".format(listToString(statElements)) ) # get all the spectra data in batch and process all the windows spectraData, globalIndices = specReader.readBinaryBatchGlobal() if options["ncores"] > 0: out = multiStatistics( options["ncores"], spectraData, evalFreq, options["stats"] ) for iW in range(numWindows): for stat in options["stats"]: statData[stat].addStat(iW, globalIndices[iW], out[iW][stat]) else: statCalculator = StatisticCalculator() for iW in range(numWindows): winSpecData = spectraData[iW] winStatData = calculateWindowStatistics( winSpecData, evalFreq, options["stats"], statCalculator=statCalculator, ) for stat in options["stats"]: statData[stat].addStat( iW, globalIndices[iW], winStatData[stat] ) specReader.closeFile() # save statistic for stat in options["stats"]: statIO.setDatapath( os.path.join( siteData.getMeasurementStatPath(meas), options["specdir"] ) ) statIO.write(statData[stat], declevel)
def viewImpedance(self, **kwargs) -> Figure: """Plots the transfer function data For resistivity data, both axes are log scale (period and resistivity). For phase data, period is in log scale and phase is linear scale. Units, x axis is seconds, resistivity is Ohm m and phase is degrees. Parameters ---------- polarisations : List[str], optional Polarisations to plot fig : matplotlib.pyplot.figure, optional A figure object oneplot : bool, optional Boolean flag for plotting all polarisations on one plot rather than separate plots colours : Dict[str, str], optional Colours dictionary for plotting impedance components mk : str, optional Plot marker type ls : str, optional Line style plotfonts : Dict, optional A dictionary of plot fonts label : str, optional Label for the plots xlim : List, optional Limits for the x axis res_ylim : List, optional Limits for the resistivity y axis phase_ylim : List, optional Limits for the phase y axis Returns ------- plt.figure Matplotlib figure object """ polarisations = ( kwargs["polarisations"] if "polarisations" in kwargs else self.polarisations ) # limits xlim = kwargs["xlim"] if "xlim" in kwargs else [1e-3, 1e4] res_ylim = kwargs["res_ylim"] if "res_ylim" in kwargs else [1e-2, 1e3] phase_ylim = kwargs["phase_ylim"] if "phase_ylim" in kwargs else [-30, 120] # markers colours = ( kwargs["colours"] if "colours" in kwargs else transferFunctionColours() ) mk = kwargs["mk"] if "mk" in kwargs else "o" ls = kwargs["ls"] if "ls" in kwargs else "none" plotfonts = kwargs["plotfonts"] if "plotfonts" in kwargs else getViewFonts() # calculate number of rows and columns oneplot = False if "oneplot" in kwargs and kwargs["oneplot"]: oneplot = True nrows = 2 ncols = 1 if oneplot else len(polarisations) # a multiplier to make sure all the components end up on the right plot plotNumMult = 0 if ncols > 1 else 1 # plot if "fig" in kwargs: fig = plt.figure(kwargs["fig"].number) else: figsize = getTransferFunctionFigSize(oneplot, len(polarisations)) fig = plt.figure(figsize=figsize) st = fig.suptitle( "Impedance tensor apparent resistivity and phase", fontsize=plotfonts["suptitle"], ) st.set_y(0.98) for idx, pol in enumerate(polarisations): res, phase = self.getResAndPhase(pol) resError, phaseError = self.getResAndPhaseErrors(pol) label = kwargs["label"] + " - {}".format(pol) if "label" in kwargs else pol # plot resistivity ax1 = plt.subplot(nrows, ncols, idx + 1 - plotNumMult * idx) # the title if not oneplot: plt.title("Polarisation {}".format(pol), fontsize=plotfonts["title"]) else: plt.title( "Polarisations {}".format(listToString(polarisations)), fontsize=plotfonts["title"], ) # plot the data ax1.errorbar( self.period, res, yerr=resError, ls=ls, marker=mk, markersize=7, markerfacecolor="white", markeredgecolor=colours[pol], mew=1.1, color=colours[pol], ecolor=colours[pol], elinewidth=1.0, capsize=4, barsabove=False, label=label, ) ax1.set_xscale("log") ax1.set_yscale("log") ax1.set_aspect("equal", adjustable="box") # axis options plt.ylabel("Apparent Res. [Ohm m]", fontsize=plotfonts["axisLabel"]) plt.xlim(xlim) plt.ylim(res_ylim) # set tick sizes for lab in ax1.get_xticklabels() + ax1.get_yticklabels(): lab.set_fontsize(plotfonts["axisTicks"]) # plot phase ax2 = plt.subplot(nrows, ncols, ncols + idx + 1 - plotNumMult * idx) # plot the data ax2.errorbar( self.period, phase, yerr=phaseError, ls="none", marker=mk, markersize=7, markerfacecolor="white", markeredgecolor=colours[pol], mew=1.1, color=colours[pol], ecolor=colours[pol], elinewidth=1.0, capsize=4, barsabove=False, label=label, ) ax2.set_xscale("log") # axis options plt.xlabel("Period [s]", fontsize=plotfonts["axisLabel"]) plt.ylabel("Phase [degrees]", fontsize=plotfonts["axisLabel"]) plt.xlim(xlim) plt.ylim(phase_ylim) # set tick sizes for lab in ax2.get_xticklabels() + ax2.get_yticklabels(): lab.set_fontsize(plotfonts["axisTicks"]) # add the legend for idx, pol in enumerate(polarisations): ax1 = plt.subplot(nrows, ncols, idx + 1 - plotNumMult * idx) leg = plt.legend(loc="lower left", fontsize=plotfonts["legend"]) leg.get_frame().set_linewidth(0.0) leg.get_frame().set_facecolor("w") plt.grid(True, ls="--") ax2 = plt.subplot(nrows, ncols, ncols + idx + 1 - plotNumMult * idx) leg = plt.legend(loc="lower left", fontsize=plotfonts["legend"]) leg.get_frame().set_linewidth(0.0) leg.get_frame().set_facecolor("w") plt.grid(True, ls="--") # show if the figure is not in keywords if "fig" not in kwargs: # layout options plt.tight_layout() fig.subplots_adjust(top=0.92) plt.show() return fig