def calculateWindowStatistics( winSpecData: SpectrumData, evalFreq: np.ndarray, stats: List[str], remoteSpecData: Union[SpectrumData, None] = None, statCalculator: Union[StatisticCalculator, None] = None, ) -> Dict[str, Any]: """Calculate the statistics for a single window This method is meant to be used with multiprocessing Parameters ---------- winSpecData : SpectrumData Spectrum data for a single window evalFreq : np.ndarray Array for evaluation frequencies stats : List[str] List of statistics to calculate remoteSpecData : None, SpectrumData Optionally add remote spectrum data to calculate remote statistics statCalculator : None, StatisticCalculator Provide a statistic calculator to avoid multiple creation of objects """ if statCalculator is None: statCalculator = StatisticCalculator() statCalculator.setSpectra(winSpecData.freqArray, winSpecData, evalFreq) if remoteSpecData is not None: statCalculator.addRemoteSpec(remoteSpecData) data = {} for stat in stats: data[stat] = statCalculator.getDataForStatName(stat) return data
def test_statistics_calculator_power_spectral_density(): """Test absolute value calculator""" from resistics.statistics.calculator import StatisticCalculator import numpy as np specData, evalfreq = get_spectrum_data() calculator = StatisticCalculator() calculator.winLen = 1 assert calculator.winLen == 1 calculator.setSpectra(specData.freqArray, specData, evalfreq) statData = calculator.getDataForStatName("powerSpectralDensity") testData = { 24: { "psdEx": 912.0, "psdEy": 640.0, "psdHx": 1496.0, "psdHy": 1680.0 }, 40: { "psdEx": 784.0, "psdEy": 480.0, "psdHx": 792.0, "psdHy": 1344.0 }, } for efreq in evalfreq: for key, val in statData[efreq].items(): np.testing.assert_almost_equal(val, testData[efreq][key])
def test_statistics_calculator_coherence(): """Test absolute value calculator""" from resistics.statistics.calculator import StatisticCalculator import numpy as np specData, evalfreq = get_spectrum_data() calculator = StatisticCalculator() calculator.winLen = 1 assert calculator.winLen == 1 calculator.setSpectra(specData.freqArray, specData, evalfreq) statData = calculator.getDataForStatName("coherence") testData = { 24: { "cohExHx": 0.5462519936204147, "cohExHy": 0.13675856307435255, "cohEyHx": 0.590909090909091, "cohEyHy": 0.19523809523809524, }, 40: { "cohExHx": 0.49360956503813647, "cohExHy": 0.6379980563654033, "cohEyHx": 0.6734006734006734, "cohEyHy": 0.20634920634920634, }, } for efreq in evalfreq: for key, val in statData[efreq].items(): np.testing.assert_almost_equal(val, testData[efreq][key])
def test_statistics_calculator_transfer_function(): """Test absolute value calculator""" from resistics.statistics.calculator import StatisticCalculator import numpy as np specData, evalfreq = get_spectrum_data() calculator = StatisticCalculator() calculator.winLen = 1 assert calculator.winLen == 1 calculator.setSpectra(specData.freqArray, specData, evalfreq) statData = calculator.getDataForStatName("transferFunction") testData = { 24: { "ExHxRes": 0.0051423310440927615, "ExHxPhase": -38.08089717250079, "ExHxReal": 0.6183338309943266, "ExHxImag": -0.484502836667662, "ExHyRes": 0.002406937394247041, "ExHyPhase": -79.49669804710025, "ExHyReal": 0.09796954314720807, "ExHyImag": -0.5284263959390865, "EyHxRes": 0.003364188314919875, "EyHxPhase": 40.70059399014801, "EyHxReal": 0.48169602866527317, "EyHxImag": 0.4143326366079426, "EyHyRes": 0.001522842639593909, "EyHyPhase": 49.044485574181074, "EyHyReal": 0.2802030456852794, "EyHyImag": 0.3228426395939085, }, 40: { "ExHxRes": 0.0021009588268471532, "ExHxPhase": -12.512585801455565, "ExHxReal": 0.6328257191201355, "ExHxImag": -0.14043993231810512, "ExHyRes": 0.0017318809926677931, "ExHyPhase": -91.1943471837543, "ExHyReal": -0.012267343485617588, "ExHyImag": -0.5884094754653127, "EyHxRes": 0.002820078962210943, "EyHxPhase": 120.6095367512591, "EyHxReal": -0.3824027072758038, "EyHxImag": 0.6463620981387479, "EyHyRes": 0.0006838691483361542, "EyHyPhase": 1.4419233716812918, "EyHyReal": 0.36971235194585467, "EyHyImag": 0.009306260575296085, }, } for efreq in evalfreq: for key, val in statData[efreq].items(): np.testing.assert_almost_equal(val, testData[efreq][key])
def test_statistics_calculator_absval(): """Test absolute value calculator""" from resistics.statistics.calculator import StatisticCalculator import numpy as np specData, evalfreq = get_spectrum_data() calculator = StatisticCalculator() calculator.winLen = 1 assert calculator.winLen == 1 calculator.setSpectra(specData.freqArray, specData, evalfreq) statData = calculator.getDataForStatName("absvalEqn") testData = { 24: { "absExHx": 53.956000593075835, "absEyHx": 47.01063709417264, "absHxHx": 93.5, "absHyHx": 38.01315561749642, "absExHy": 28.609439001839934, "absEyHy": 28.635642126552707, "absHxHy": 38.01315561749642, "absHyHy": 105.0, "absExEx": 57.0, "absEyEx": 40.0, "absHxEx": 53.956000593075835, "absHyEx": 28.609439001839934, "absExEy": 40.0, "absEyEy": 40.0, "absHxEy": 47.01063709417264, "absHyEy": 28.635642126552707, }, 40: { "absExHx": 34.60130055359191, "absEyHx": 31.622776601683793, "absHxHx": 49.5, "absHyHx": 24.73863375370596, "absExHy": 51.24451190127583, "absEyHy": 22.80350850198276, "absHxHy": 24.73863375370596, "absHyHy": 84.0, "absExEx": 49.0, "absEyEx": 33.83784863137726, "absHxEx": 34.60130055359191, "absHyEx": 51.24451190127583, "absExEy": 33.83784863137726, "absEyEy": 30.0, "absHxEy": 31.622776601683793, "absHyEy": 22.80350850198276, }, } for efreq in evalfreq: for key, val in statData[efreq].items(): np.testing.assert_almost_equal(val, testData[efreq][key])
def test_statistics_calculator_partial_coherence(): """Test absolute value calculator""" from resistics.statistics.calculator import StatisticCalculator import numpy as np specData, evalfreq = get_spectrum_data() calculator = StatisticCalculator() calculator.winLen = 1 assert calculator.winLen == 1 calculator.setSpectra(specData.freqArray, specData, evalfreq) statData = calculator.getDataForStatName("partialCoherence") testData = { 24: { "cohExHx": 0.5462519936204147, "cohExHy": 0.13675856307435255, "cohEyHx": 0.590909090909091, "cohEyHy": 0.19523809523809524, "bivarEx": (1 + 0j), "parExHx": (1 + 0j), "parExHy": (1 + 0j), "bivarEy": (1 + 4.4408920985006264e-17j), "parEyHx": (1 + 5.518268288077701e-17j), "parEyHy": (0.9999999999999999 + 1.085551401855709e-16j), }, 40: { "cohExHx": 0.49360956503813647, "cohExHy": 0.6379980563654033, "cohEyHx": 0.6734006734006734, "cohEyHy": 0.20634920634920634, "bivarEx": (1 + 0j), "parExHx": (1 + 0j), "parExHy": (1 + 0j), "bivarEy": (1 + 2.960594732333751e-17j), "parEyHx": (0.9999999999999999 + 3.7303493627405256e-17j), "parEyHy": (0.9999999999999999 + 9.064913768073444e-17j), }, } for efreq in evalfreq: for key, val in statData[efreq].items(): np.testing.assert_almost_equal(val, testData[efreq][key])
def test_statistics_calculator_polarisation_direction(): """Test absolute value calculator""" from resistics.statistics.calculator import StatisticCalculator import numpy as np specData, evalfreq = get_spectrum_data() calculator = StatisticCalculator() calculator.winLen = 1 assert calculator.winLen == 1 calculator.setSpectra(specData.freqArray, specData, evalfreq) statData = calculator.getDataForStatName("polarisationDirection") testData = { 24: { "polExEy": 0.0, "polHxHy": 80.4010969314582 }, 40: { "polExEy": -63.43494882292201, "polHxHy": -54.293308599397115 }, } for efreq in evalfreq: for key, val in statData[efreq].items(): np.testing.assert_almost_equal(val, testData[efreq][key])
def test_statistics_calculator() -> None: """Test initialising the statistics calculator""" from resistics.statistics.calculator import StatisticCalculator import numpy as np calculator = StatisticCalculator() assert calculator.inChans == ["Hx", "Hy"] assert calculator.inSize == 2 assert calculator.outChans == ["Ex", "Ey"] assert calculator.outSize == 2 assert calculator.specChans == ["Hx", "Hy", "Ex", "Ey"] assert calculator.remoteChans == ["Hx", "Hy"] assert calculator.psdChans == ["Ex", "Ey", "Hx", "Hy"] assert calculator.cohPairs == [ ["Ex", "Hx"], ["Ex", "Hy"], ["Ey", "Hx"], ["Ey", "Hy"], ] assert calculator.polDirs == [["Ex", "Ey"], ["Hx", "Hy"]] specData, evalfreq = get_spectrum_data() calculator.winLen = 1 assert calculator.winLen == 1 calculator.setSpectra(specData.freqArray, specData, evalfreq) # expected results powerDict = { "Hx-Hx": [18.0 + 0.0j, 170.0 + 0.0j, 17.0 + 0.0j, 82.0 + 0.0j, 8.0 + 0.0j], "Hx-Hy": [21.0 + 33.0j, -92.0 + 74.0j, 24.0 - 40.0j, 24.0 + 52.0j, 14.0 + 6.0j], "Hx-Ex": [6.0 + 12.0j, 57.0 + 41.0j, 34.0 + 17.0j, 21.0 + 25.0j, 24.0 - 8.0j], "Hx-Ey": [ -48.0 + 24.0j, 80.0 - 20.0j, 2.0 - 26.0j, -22.0 - 34.0j, -24.0 + 0.0j, ], "Hy-Hx": [21.0 - 33.0j, -92.0 - 74.0j, 24.0 + 40.0j, 24.0 - 52.0j, 14.0 - 6.0j], "Hy-Hy": [85.0 + 0.0j, 82.0 + 0.0j, 128.0 + 0.0j, 40.0 + 0.0j, 29.0 + 0.0j], "Hy-Ex": [29.0 + 3.0j, -13.0 - 47.0j, 8.0 + 104.0j, 22.0 - 6.0j, 36.0 - 32.0j], "Hy-Ey": [ -12.0 + 116.0j, -52.0 - 24.0j, 64.0 - 32.0j, -28.0 + 4.0j, -42.0 + 18.0j, ], "Ex-Hx": [6.0 - 12.0j, 57.0 - 41.0j, 34.0 - 17.0j, 21.0 - 25.0j, 24.0 + 8.0j], "Ex-Hy": [29.0 - 3.0j, -13.0 + 47.0j, 8.0 - 104.0j, 22.0 + 6.0j, 36.0 + 32.0j], "Ex-Ex": [10.0 + 0.0j, 29.0 + 0.0j, 85.0 + 0.0j, 13.0 + 0.0j, 80.0 + 0.0j], "Ex-Ey": [ 0.0 + 40.0j, 22.0 - 26.0j, -22.0 - 54.0j, -16.0 - 2.0j, -72.0 - 24.0j, ], "Ey-Hx": [ -48.0 - 24.0j, 80.0 + 20.0j, 2.0 + 26.0j, -22.0 + 34.0j, -24.0 - 0.0j, ], "Ey-Hy": [ -12.0 - 116.0j, -52.0 + 24.0j, 64.0 + 32.0j, -28.0 - 4.0j, -42.0 - 18.0j, ], "Ey-Ex": [ 0.0 - 40.0j, 22.0 + 26.0j, -22.0 + 54.0j, -16.0 + 2.0j, -72.0 + 24.0j, ], "Ey-Ey": [160.0 + 0.0j, 40.0 + 0.0j, 40.0 + 0.0j, 20.0 + 0.0j, 72.0 + 0.0j], } evalDict = { "Hx-Hx": np.array([93.5 + 0.0j, 49.5 + 0.0j]), "Hx-Hy": np.array([-34.0 + 17.0j, 24.0 + 6.0j]), "Hx-Ex": np.array([45.5 + 29.0j, 27.5 + 21.0j]), "Hx-Ey": np.array([41.0 - 23.0j, -10.0 - 30.0j]), "Hy-Hx": np.array([-34.0 - 17.0j, 24.0 - 6.0j]), "Hy-Hy": np.array([105.0 + 0.0j, 84.0 + 0.0j]), "Hy-Ex": np.array([-2.5 + 28.5j, 15.0 + 49.0j]), "Hy-Ey": np.array([6.0 - 28.0j, 18.0 - 14.0j]), "Ex-Hx": np.array([45.5 - 29.0j, 27.5 - 21.0j]), "Ex-Hy": np.array([-2.5 - 28.5j, 15.0 - 49.0j]), "Ex-Ex": np.array([57.0 + 0.0j, 49.0 + 0.0j]), "Ex-Ey": np.array([0.0 - 40.0j, -19.0 - 28.0j]), "Ey-Hx": np.array([41.0 + 23.0j, -10.0 + 30.0j]), "Ey-Hy": np.array([6.0 + 28.0j, 18.0 + 14.0j]), "Ey-Ex": np.array([0.0 + 40.0j, -19.0 + 28.0j]), "Ey-Ey": np.array([40.0 + 0.0j, 30.0 + 0.0j]), } # check the autopower data for key in powerDict: splitkey = key.split("-") chan1 = splitkey[0] chan2 = splitkey[1] np.testing.assert_almost_equal( calculator.xpowers.getPower(chan1, chan2), np.array(powerDict[key])) np.testing.assert_almost_equal( calculator.xpowersEval.getPower(chan1, chan2), evalDict[key])
def calculateRemoteStatistics(projData: ProjectData, remoteSite: str, **kwargs): """Calculate statistics involving a remote reference site Parameters ---------- projData : ProjectData A project data instance remoteSite : str The name of the site to use as the remote site sites : List[str], optional A list of sites to calculate statistics for sampleFreqs : List[float], optional List of sampling frequencies for which to calculate statistics specdir : str, optional The spectra directory for which to calculate statistics remotestats : List[str], optional The statistics to calculate out. Acceptable statistics are: "RR_coherence", "RR_coherenceEqn", "RR_absvalEqn", "RR_transferFunction", "RR_resPhase". Configuration file values are used by default. """ from resistics.statistics.io import StatisticIO from resistics.statistics.calculator import StatisticCalculator from resistics.project.shortcuts import ( getDecimationParameters, getWindowParameters, getWindowSelector, ) options = {} options["sites"] = projData.getSites() options["sampleFreqs"] = projData.getSampleFreqs() options["chans"] = [] options["specdir"] = projData.config.configParams["Spectra"]["specdir"] options["remotestats"] = projData.config.configParams["Statistics"]["remotestats"] options["ncores"] = projData.config.getStatisticCores() options = parseKeywords(options, kwargs) projectText( "Calculating stats: {} for sites: {} with remote site {}".format( listToString(options["remotestats"]), listToString(options["sites"]), remoteSite, ) ) statIO = StatisticIO() for site in options["sites"]: siteData = projData.getSiteData(site) measurements = siteData.getMeasurements() for meas in measurements: sampleFreq = siteData.getMeasurementSampleFreq(meas) if sampleFreq not in options["sampleFreqs"]: continue projectText( "Calculating stats for site {}, measurement {} with reference {}".format( site, meas, remoteSite ) ) # decimation and window parameters decParams = getDecimationParameters(sampleFreq, projData.config) numLevels = decParams.numLevels winParams = getWindowParameters(decParams, projData.config) # create the window selector and find the shared windows winSelector = getWindowSelector(projData, decParams, winParams) winSelector.setSites([site, remoteSite]) winSelector.calcSharedWindows() # create the spectrum reader specReader = SpectrumReader( os.path.join(siteData.getMeasurementSpecPath(meas), options["specdir"]) ) # calculate statistics for decimation level if spectra file exists for declevel in range(0, numLevels): check = specReader.openBinaryForReading("spectra", declevel) if not check: continue # information regarding only this spectra file refTime = specReader.getReferenceTime() winSize = specReader.getWindowSize() winOlap = specReader.getWindowOverlap() numWindows = specReader.getNumWindows() evalFreq = decParams.getEvalFrequenciesForLevel(declevel) sampleFreqDec = specReader.getSampleFreq() globalOffset = specReader.getGlobalOffset() # find size of the intersection between the windows in this spectra file and the shared windows sharedWindows = winSelector.getSharedWindowsLevel(declevel) sharedWindowsMeas = sharedWindows.intersection( set(np.arange(globalOffset, globalOffset + numWindows)) ) sharedWindowsMeas = sorted(list(sharedWindowsMeas)) numSharedWindows = len(sharedWindowsMeas) statData = {} # create the statistic handlers for stat in options["remotestats"]: statElements = getStatElements(stat) statData[stat] = StatisticData( stat, refTime, sampleFreqDec, winSize, winOlap ) # with remote reference the number of windows is number of shared windows statData[stat].setStatParams( numSharedWindows, statElements, evalFreq ) statData[stat].comments = specReader.getComments() statData[stat].addComment(projData.config.getConfigComment()) statData[stat].addComment( "Calculating remote statistic: {}".format(stat) ) statData[stat].addComment( "Statistic components: {}".format(listToString(statElements)) ) # collect the spectra data spectraData, _globalIndices = specReader.readBinaryBatchGlobal( sharedWindowsMeas ) remoteData = [] for globalWindow in sharedWindowsMeas: _, remoteReader = winSelector.getSpecReaderForWindow( remoteSite, declevel, globalWindow ) remoteData.append(remoteReader.readBinaryWindowGlobal(globalWindow)) # calculate if options["ncores"] > 0: out = multiStatistics( options["ncores"], spectraData, evalFreq, options["remotestats"], remoteData=remoteData, ) for iW, globalWindow in enumerate(sharedWindowsMeas): for stat in options["remotestats"]: statData[stat].addStat(iW, globalWindow, out[iW][stat]) else: statCalculator = StatisticCalculator() for iW, globalWindow in enumerate(sharedWindowsMeas): winStatData = calculateWindowStatistics( spectraData[iW], evalFreq, options["remotestats"], remoteSpecData=remoteData[iW], statCalculator=statCalculator, ) for stat in options["remotestats"]: statData[stat].addStat(iW, globalWindow, winStatData[stat]) # save statistic for stat in options["remotestats"]: statIO.setDatapath( os.path.join( siteData.getMeasurementStatPath(meas), options["specdir"] ) ) statIO.write(statData[stat], declevel)
def calculateStatistics(projData: ProjectData, **kwargs): """Calculate statistics for sites Parameters ---------- projData : ProjectData A project data instance sites : List[str], optional A list of sites to calculate statistics for sampleFreqs : List[float], optional List of sampling frequencies for which to calculate statistics specdir : str, optional The spectra directory for which to calculate statistics stats : List[str], optional The statistics to calculate out. Acceptable values are: "absvalEqn" "coherence", "psd", "poldir", "transFunc", "resPhase", "partialcoh". Configuration file values are used by default. ncores : int, optional The number of cores to run the transfer function calculations on """ from resistics.statistics.io import StatisticIO from resistics.project.shortcuts import getDecimationParameters options = {} options["sites"] = projData.getSites() options["sampleFreqs"] = projData.getSampleFreqs() options["chans"] = [] options["specdir"] = projData.config.configParams["Spectra"]["specdir"] options["stats"] = projData.config.configParams["Statistics"]["stats"] options["ncores"] = projData.config.getStatisticCores() options = parseKeywords(options, kwargs) projectText( "Calculating stats: {} for sites: {}".format( listToString(options["stats"]), listToString(options["sites"]) ) ) # loop through sites and calculate statistics statIO = StatisticIO() for site in options["sites"]: siteData = projData.getSiteData(site) measurements = siteData.getMeasurements() for meas in measurements: sampleFreq = siteData.getMeasurementSampleFreq(meas) if sampleFreq not in options["sampleFreqs"]: continue projectText( "Calculating stats for site {}, measurement {}".format(site, meas) ) decParams = getDecimationParameters(sampleFreq, projData.config) numLevels = decParams.numLevels specReader = SpectrumReader( os.path.join(siteData.getMeasurementSpecPath(meas), options["specdir"]) ) # calculate statistics for decimation level if spectra file exists for declevel in range(0, numLevels): check = specReader.openBinaryForReading("spectra", declevel) if not check: continue refTime = specReader.getReferenceTime() winSize = specReader.getWindowSize() winOlap = specReader.getWindowOverlap() numWindows = specReader.getNumWindows() sampleFreqDec = specReader.getSampleFreq() evalFreq = decParams.getEvalFrequenciesForLevel(declevel) # dictionary for saving statistic data statData = {} for stat in options["stats"]: statElements = getStatElements(stat) statData[stat] = StatisticData( stat, refTime, sampleFreqDec, winSize, winOlap ) statData[stat].setStatParams(numWindows, statElements, evalFreq) statData[stat].comments = specReader.getComments() statData[stat].addComment(projData.config.getConfigComment()) statData[stat].addComment("Calculating statistic: {}".format(stat)) statData[stat].addComment( "Statistic components: {}".format(listToString(statElements)) ) # get all the spectra data in batch and process all the windows spectraData, globalIndices = specReader.readBinaryBatchGlobal() if options["ncores"] > 0: out = multiStatistics( options["ncores"], spectraData, evalFreq, options["stats"] ) for iW in range(numWindows): for stat in options["stats"]: statData[stat].addStat(iW, globalIndices[iW], out[iW][stat]) else: statCalculator = StatisticCalculator() for iW in range(numWindows): winSpecData = spectraData[iW] winStatData = calculateWindowStatistics( winSpecData, evalFreq, options["stats"], statCalculator=statCalculator, ) for stat in options["stats"]: statData[stat].addStat( iW, globalIndices[iW], winStatData[stat] ) specReader.closeFile() # save statistic for stat in options["stats"]: statIO.setDatapath( os.path.join( siteData.getMeasurementStatPath(meas), options["specdir"] ) ) statIO.write(statData[stat], declevel)