def calculateWindowStatistics(
winSpecData: SpectrumData,
evalFreq: np.ndarray,
stats: List[str],
remoteSpecData: Union[SpectrumData, None] = None,
statCalculator: Union[StatisticCalculator, None] = None,
) -> Dict[str, Any]:
"""Calculate the statistics for a single window
This method is meant to be used with multiprocessing
Parameters
----------
winSpecData : SpectrumData
Spectrum data for a single window
evalFreq : np.ndarray
Array for evaluation frequencies
stats : List[str]
List of statistics to calculate
remoteSpecData : None, SpectrumData
Optionally add remote spectrum data to calculate remote statistics
statCalculator : None, StatisticCalculator
Provide a statistic calculator to avoid multiple creation of objects
"""
if statCalculator is None:
statCalculator = StatisticCalculator()
statCalculator.setSpectra(winSpecData.freqArray, winSpecData, evalFreq)
if remoteSpecData is not None:
statCalculator.addRemoteSpec(remoteSpecData)
data = {}
for stat in stats:
data[stat] = statCalculator.getDataForStatName(stat)
return data
def test_statistics_calculator_power_spectral_density():
"""Test absolute value calculator"""
from resistics.statistics.calculator import StatisticCalculator
import numpy as np
specData, evalfreq = get_spectrum_data()
calculator = StatisticCalculator()
calculator.winLen = 1
assert calculator.winLen == 1
calculator.setSpectra(specData.freqArray, specData, evalfreq)
statData = calculator.getDataForStatName("powerSpectralDensity")
testData = {
24: {
"psdEx": 912.0,
"psdEy": 640.0,
"psdHx": 1496.0,
"psdHy": 1680.0
},
40: {
"psdEx": 784.0,
"psdEy": 480.0,
"psdHx": 792.0,
"psdHy": 1344.0
},
}
for efreq in evalfreq:
for key, val in statData[efreq].items():
np.testing.assert_almost_equal(val, testData[efreq][key])
def test_statistics_calculator_coherence():
"""Test absolute value calculator"""
from resistics.statistics.calculator import StatisticCalculator
import numpy as np
specData, evalfreq = get_spectrum_data()
calculator = StatisticCalculator()
calculator.winLen = 1
assert calculator.winLen == 1
calculator.setSpectra(specData.freqArray, specData, evalfreq)
statData = calculator.getDataForStatName("coherence")
testData = {
24: {
"cohExHx": 0.5462519936204147,
"cohExHy": 0.13675856307435255,
"cohEyHx": 0.590909090909091,
"cohEyHy": 0.19523809523809524,
},
40: {
"cohExHx": 0.49360956503813647,
"cohExHy": 0.6379980563654033,
"cohEyHx": 0.6734006734006734,
"cohEyHy": 0.20634920634920634,
},
}
for efreq in evalfreq:
for key, val in statData[efreq].items():
np.testing.assert_almost_equal(val, testData[efreq][key])
def test_statistics_calculator_transfer_function():
"""Test absolute value calculator"""
from resistics.statistics.calculator import StatisticCalculator
import numpy as np
specData, evalfreq = get_spectrum_data()
calculator = StatisticCalculator()
calculator.winLen = 1
assert calculator.winLen == 1
calculator.setSpectra(specData.freqArray, specData, evalfreq)
statData = calculator.getDataForStatName("transferFunction")
testData = {
24: {
"ExHxRes": 0.0051423310440927615,
"ExHxPhase": -38.08089717250079,
"ExHxReal": 0.6183338309943266,
"ExHxImag": -0.484502836667662,
"ExHyRes": 0.002406937394247041,
"ExHyPhase": -79.49669804710025,
"ExHyReal": 0.09796954314720807,
"ExHyImag": -0.5284263959390865,
"EyHxRes": 0.003364188314919875,
"EyHxPhase": 40.70059399014801,
"EyHxReal": 0.48169602866527317,
"EyHxImag": 0.4143326366079426,
"EyHyRes": 0.001522842639593909,
"EyHyPhase": 49.044485574181074,
"EyHyReal": 0.2802030456852794,
"EyHyImag": 0.3228426395939085,
},
40: {
"ExHxRes": 0.0021009588268471532,
"ExHxPhase": -12.512585801455565,
"ExHxReal": 0.6328257191201355,
"ExHxImag": -0.14043993231810512,
"ExHyRes": 0.0017318809926677931,
"ExHyPhase": -91.1943471837543,
"ExHyReal": -0.012267343485617588,
"ExHyImag": -0.5884094754653127,
"EyHxRes": 0.002820078962210943,
"EyHxPhase": 120.6095367512591,
"EyHxReal": -0.3824027072758038,
"EyHxImag": 0.6463620981387479,
"EyHyRes": 0.0006838691483361542,
"EyHyPhase": 1.4419233716812918,
"EyHyReal": 0.36971235194585467,
"EyHyImag": 0.009306260575296085,
},
}
for efreq in evalfreq:
for key, val in statData[efreq].items():
np.testing.assert_almost_equal(val, testData[efreq][key])
def test_statistics_calculator_absval():
"""Test absolute value calculator"""
from resistics.statistics.calculator import StatisticCalculator
import numpy as np
specData, evalfreq = get_spectrum_data()
calculator = StatisticCalculator()
calculator.winLen = 1
assert calculator.winLen == 1
calculator.setSpectra(specData.freqArray, specData, evalfreq)
statData = calculator.getDataForStatName("absvalEqn")
testData = {
24: {
"absExHx": 53.956000593075835,
"absEyHx": 47.01063709417264,
"absHxHx": 93.5,
"absHyHx": 38.01315561749642,
"absExHy": 28.609439001839934,
"absEyHy": 28.635642126552707,
"absHxHy": 38.01315561749642,
"absHyHy": 105.0,
"absExEx": 57.0,
"absEyEx": 40.0,
"absHxEx": 53.956000593075835,
"absHyEx": 28.609439001839934,
"absExEy": 40.0,
"absEyEy": 40.0,
"absHxEy": 47.01063709417264,
"absHyEy": 28.635642126552707,
},
40: {
"absExHx": 34.60130055359191,
"absEyHx": 31.622776601683793,
"absHxHx": 49.5,
"absHyHx": 24.73863375370596,
"absExHy": 51.24451190127583,
"absEyHy": 22.80350850198276,
"absHxHy": 24.73863375370596,
"absHyHy": 84.0,
"absExEx": 49.0,
"absEyEx": 33.83784863137726,
"absHxEx": 34.60130055359191,
"absHyEx": 51.24451190127583,
"absExEy": 33.83784863137726,
"absEyEy": 30.0,
"absHxEy": 31.622776601683793,
"absHyEy": 22.80350850198276,
},
}
for efreq in evalfreq:
for key, val in statData[efreq].items():
np.testing.assert_almost_equal(val, testData[efreq][key])
def test_statistics_calculator_partial_coherence():
"""Test absolute value calculator"""
from resistics.statistics.calculator import StatisticCalculator
import numpy as np
specData, evalfreq = get_spectrum_data()
calculator = StatisticCalculator()
calculator.winLen = 1
assert calculator.winLen == 1
calculator.setSpectra(specData.freqArray, specData, evalfreq)
statData = calculator.getDataForStatName("partialCoherence")
testData = {
24: {
"cohExHx": 0.5462519936204147,
"cohExHy": 0.13675856307435255,
"cohEyHx": 0.590909090909091,
"cohEyHy": 0.19523809523809524,
"bivarEx": (1 + 0j),
"parExHx": (1 + 0j),
"parExHy": (1 + 0j),
"bivarEy": (1 + 4.4408920985006264e-17j),
"parEyHx": (1 + 5.518268288077701e-17j),
"parEyHy": (0.9999999999999999 + 1.085551401855709e-16j),
},
40: {
"cohExHx": 0.49360956503813647,
"cohExHy": 0.6379980563654033,
"cohEyHx": 0.6734006734006734,
"cohEyHy": 0.20634920634920634,
"bivarEx": (1 + 0j),
"parExHx": (1 + 0j),
"parExHy": (1 + 0j),
"bivarEy": (1 + 2.960594732333751e-17j),
"parEyHx": (0.9999999999999999 + 3.7303493627405256e-17j),
"parEyHy": (0.9999999999999999 + 9.064913768073444e-17j),
},
}
for efreq in evalfreq:
for key, val in statData[efreq].items():
np.testing.assert_almost_equal(val, testData[efreq][key])
def test_statistics_calculator_polarisation_direction():
"""Test absolute value calculator"""
from resistics.statistics.calculator import StatisticCalculator
import numpy as np
specData, evalfreq = get_spectrum_data()
calculator = StatisticCalculator()
calculator.winLen = 1
assert calculator.winLen == 1
calculator.setSpectra(specData.freqArray, specData, evalfreq)
statData = calculator.getDataForStatName("polarisationDirection")
testData = {
24: {
"polExEy": 0.0,
"polHxHy": 80.4010969314582
},
40: {
"polExEy": -63.43494882292201,
"polHxHy": -54.293308599397115
},
}
for efreq in evalfreq:
for key, val in statData[efreq].items():
np.testing.assert_almost_equal(val, testData[efreq][key])
def test_statistics_calculator() -> None:
"""Test initialising the statistics calculator"""
from resistics.statistics.calculator import StatisticCalculator
import numpy as np
calculator = StatisticCalculator()
assert calculator.inChans == ["Hx", "Hy"]
assert calculator.inSize == 2
assert calculator.outChans == ["Ex", "Ey"]
assert calculator.outSize == 2
assert calculator.specChans == ["Hx", "Hy", "Ex", "Ey"]
assert calculator.remoteChans == ["Hx", "Hy"]
assert calculator.psdChans == ["Ex", "Ey", "Hx", "Hy"]
assert calculator.cohPairs == [
["Ex", "Hx"],
["Ex", "Hy"],
["Ey", "Hx"],
["Ey", "Hy"],
]
assert calculator.polDirs == [["Ex", "Ey"], ["Hx", "Hy"]]
specData, evalfreq = get_spectrum_data()
calculator.winLen = 1
assert calculator.winLen == 1
calculator.setSpectra(specData.freqArray, specData, evalfreq)
# expected results
powerDict = {
"Hx-Hx":
[18.0 + 0.0j, 170.0 + 0.0j, 17.0 + 0.0j, 82.0 + 0.0j, 8.0 + 0.0j],
"Hx-Hy":
[21.0 + 33.0j, -92.0 + 74.0j, 24.0 - 40.0j, 24.0 + 52.0j, 14.0 + 6.0j],
"Hx-Ex":
[6.0 + 12.0j, 57.0 + 41.0j, 34.0 + 17.0j, 21.0 + 25.0j, 24.0 - 8.0j],
"Hx-Ey": [
-48.0 + 24.0j,
80.0 - 20.0j,
2.0 - 26.0j,
-22.0 - 34.0j,
-24.0 + 0.0j,
],
"Hy-Hx":
[21.0 - 33.0j, -92.0 - 74.0j, 24.0 + 40.0j, 24.0 - 52.0j, 14.0 - 6.0j],
"Hy-Hy":
[85.0 + 0.0j, 82.0 + 0.0j, 128.0 + 0.0j, 40.0 + 0.0j, 29.0 + 0.0j],
"Hy-Ex":
[29.0 + 3.0j, -13.0 - 47.0j, 8.0 + 104.0j, 22.0 - 6.0j, 36.0 - 32.0j],
"Hy-Ey": [
-12.0 + 116.0j,
-52.0 - 24.0j,
64.0 - 32.0j,
-28.0 + 4.0j,
-42.0 + 18.0j,
],
"Ex-Hx":
[6.0 - 12.0j, 57.0 - 41.0j, 34.0 - 17.0j, 21.0 - 25.0j, 24.0 + 8.0j],
"Ex-Hy":
[29.0 - 3.0j, -13.0 + 47.0j, 8.0 - 104.0j, 22.0 + 6.0j, 36.0 + 32.0j],
"Ex-Ex":
[10.0 + 0.0j, 29.0 + 0.0j, 85.0 + 0.0j, 13.0 + 0.0j, 80.0 + 0.0j],
"Ex-Ey": [
0.0 + 40.0j,
22.0 - 26.0j,
-22.0 - 54.0j,
-16.0 - 2.0j,
-72.0 - 24.0j,
],
"Ey-Hx": [
-48.0 - 24.0j,
80.0 + 20.0j,
2.0 + 26.0j,
-22.0 + 34.0j,
-24.0 - 0.0j,
],
"Ey-Hy": [
-12.0 - 116.0j,
-52.0 + 24.0j,
64.0 + 32.0j,
-28.0 - 4.0j,
-42.0 - 18.0j,
],
"Ey-Ex": [
0.0 - 40.0j,
22.0 + 26.0j,
-22.0 + 54.0j,
-16.0 + 2.0j,
-72.0 + 24.0j,
],
"Ey-Ey":
[160.0 + 0.0j, 40.0 + 0.0j, 40.0 + 0.0j, 20.0 + 0.0j, 72.0 + 0.0j],
}
evalDict = {
"Hx-Hx": np.array([93.5 + 0.0j, 49.5 + 0.0j]),
"Hx-Hy": np.array([-34.0 + 17.0j, 24.0 + 6.0j]),
"Hx-Ex": np.array([45.5 + 29.0j, 27.5 + 21.0j]),
"Hx-Ey": np.array([41.0 - 23.0j, -10.0 - 30.0j]),
"Hy-Hx": np.array([-34.0 - 17.0j, 24.0 - 6.0j]),
"Hy-Hy": np.array([105.0 + 0.0j, 84.0 + 0.0j]),
"Hy-Ex": np.array([-2.5 + 28.5j, 15.0 + 49.0j]),
"Hy-Ey": np.array([6.0 - 28.0j, 18.0 - 14.0j]),
"Ex-Hx": np.array([45.5 - 29.0j, 27.5 - 21.0j]),
"Ex-Hy": np.array([-2.5 - 28.5j, 15.0 - 49.0j]),
"Ex-Ex": np.array([57.0 + 0.0j, 49.0 + 0.0j]),
"Ex-Ey": np.array([0.0 - 40.0j, -19.0 - 28.0j]),
"Ey-Hx": np.array([41.0 + 23.0j, -10.0 + 30.0j]),
"Ey-Hy": np.array([6.0 + 28.0j, 18.0 + 14.0j]),
"Ey-Ex": np.array([0.0 + 40.0j, -19.0 + 28.0j]),
"Ey-Ey": np.array([40.0 + 0.0j, 30.0 + 0.0j]),
}
# check the autopower data
for key in powerDict:
splitkey = key.split("-")
chan1 = splitkey[0]
chan2 = splitkey[1]
np.testing.assert_almost_equal(
calculator.xpowers.getPower(chan1, chan2),
np.array(powerDict[key]))
np.testing.assert_almost_equal(
calculator.xpowersEval.getPower(chan1, chan2), evalDict[key])
def calculateRemoteStatistics(projData: ProjectData, remoteSite: str, **kwargs):
"""Calculate statistics involving a remote reference site
Parameters
----------
projData : ProjectData
A project data instance
remoteSite : str
The name of the site to use as the remote site
sites : List[str], optional
A list of sites to calculate statistics for
sampleFreqs : List[float], optional
List of sampling frequencies for which to calculate statistics
specdir : str, optional
The spectra directory for which to calculate statistics
remotestats : List[str], optional
The statistics to calculate out. Acceptable statistics are: "RR_coherence", "RR_coherenceEqn", "RR_absvalEqn", "RR_transferFunction", "RR_resPhase". Configuration file values are used by default.
"""
from resistics.statistics.io import StatisticIO
from resistics.statistics.calculator import StatisticCalculator
from resistics.project.shortcuts import (
getDecimationParameters,
getWindowParameters,
getWindowSelector,
)
options = {}
options["sites"] = projData.getSites()
options["sampleFreqs"] = projData.getSampleFreqs()
options["chans"] = []
options["specdir"] = projData.config.configParams["Spectra"]["specdir"]
options["remotestats"] = projData.config.configParams["Statistics"]["remotestats"]
options["ncores"] = projData.config.getStatisticCores()
options = parseKeywords(options, kwargs)
projectText(
"Calculating stats: {} for sites: {} with remote site {}".format(
listToString(options["remotestats"]),
listToString(options["sites"]),
remoteSite,
)
)
statIO = StatisticIO()
for site in options["sites"]:
siteData = projData.getSiteData(site)
measurements = siteData.getMeasurements()
for meas in measurements:
sampleFreq = siteData.getMeasurementSampleFreq(meas)
if sampleFreq not in options["sampleFreqs"]:
continue
projectText(
"Calculating stats for site {}, measurement {} with reference {}".format(
site, meas, remoteSite
)
)
# decimation and window parameters
decParams = getDecimationParameters(sampleFreq, projData.config)
numLevels = decParams.numLevels
winParams = getWindowParameters(decParams, projData.config)
# create the window selector and find the shared windows
winSelector = getWindowSelector(projData, decParams, winParams)
winSelector.setSites([site, remoteSite])
winSelector.calcSharedWindows()
# create the spectrum reader
specReader = SpectrumReader(
os.path.join(siteData.getMeasurementSpecPath(meas), options["specdir"])
)
# calculate statistics for decimation level if spectra file exists
for declevel in range(0, numLevels):
check = specReader.openBinaryForReading("spectra", declevel)
if not check:
continue
# information regarding only this spectra file
refTime = specReader.getReferenceTime()
winSize = specReader.getWindowSize()
winOlap = specReader.getWindowOverlap()
numWindows = specReader.getNumWindows()
evalFreq = decParams.getEvalFrequenciesForLevel(declevel)
sampleFreqDec = specReader.getSampleFreq()
globalOffset = specReader.getGlobalOffset()
# find size of the intersection between the windows in this spectra file and the shared windows
sharedWindows = winSelector.getSharedWindowsLevel(declevel)
sharedWindowsMeas = sharedWindows.intersection(
set(np.arange(globalOffset, globalOffset + numWindows))
)
sharedWindowsMeas = sorted(list(sharedWindowsMeas))
numSharedWindows = len(sharedWindowsMeas)
statData = {}
# create the statistic handlers
for stat in options["remotestats"]:
statElements = getStatElements(stat)
statData[stat] = StatisticData(
stat, refTime, sampleFreqDec, winSize, winOlap
)
# with remote reference the number of windows is number of shared windows
statData[stat].setStatParams(
numSharedWindows, statElements, evalFreq
)
statData[stat].comments = specReader.getComments()
statData[stat].addComment(projData.config.getConfigComment())
statData[stat].addComment(
"Calculating remote statistic: {}".format(stat)
)
statData[stat].addComment(
"Statistic components: {}".format(listToString(statElements))
)
# collect the spectra data
spectraData, _globalIndices = specReader.readBinaryBatchGlobal(
sharedWindowsMeas
)
remoteData = []
for globalWindow in sharedWindowsMeas:
_, remoteReader = winSelector.getSpecReaderForWindow(
remoteSite, declevel, globalWindow
)
remoteData.append(remoteReader.readBinaryWindowGlobal(globalWindow))
# calculate
if options["ncores"] > 0:
out = multiStatistics(
options["ncores"],
spectraData,
evalFreq,
options["remotestats"],
remoteData=remoteData,
)
for iW, globalWindow in enumerate(sharedWindowsMeas):
for stat in options["remotestats"]:
statData[stat].addStat(iW, globalWindow, out[iW][stat])
else:
statCalculator = StatisticCalculator()
for iW, globalWindow in enumerate(sharedWindowsMeas):
winStatData = calculateWindowStatistics(
spectraData[iW],
evalFreq,
options["remotestats"],
remoteSpecData=remoteData[iW],
statCalculator=statCalculator,
)
for stat in options["remotestats"]:
statData[stat].addStat(iW, globalWindow, winStatData[stat])
# save statistic
for stat in options["remotestats"]:
statIO.setDatapath(
os.path.join(
siteData.getMeasurementStatPath(meas), options["specdir"]
)
)
statIO.write(statData[stat], declevel)
def calculateStatistics(projData: ProjectData, **kwargs):
"""Calculate statistics for sites
Parameters
----------
projData : ProjectData
A project data instance
sites : List[str], optional
A list of sites to calculate statistics for
sampleFreqs : List[float], optional
List of sampling frequencies for which to calculate statistics
specdir : str, optional
The spectra directory for which to calculate statistics
stats : List[str], optional
The statistics to calculate out. Acceptable values are: "absvalEqn" "coherence", "psd", "poldir", "transFunc", "resPhase", "partialcoh". Configuration file values are used by default.
ncores : int, optional
The number of cores to run the transfer function calculations on
"""
from resistics.statistics.io import StatisticIO
from resistics.project.shortcuts import getDecimationParameters
options = {}
options["sites"] = projData.getSites()
options["sampleFreqs"] = projData.getSampleFreqs()
options["chans"] = []
options["specdir"] = projData.config.configParams["Spectra"]["specdir"]
options["stats"] = projData.config.configParams["Statistics"]["stats"]
options["ncores"] = projData.config.getStatisticCores()
options = parseKeywords(options, kwargs)
projectText(
"Calculating stats: {} for sites: {}".format(
listToString(options["stats"]), listToString(options["sites"])
)
)
# loop through sites and calculate statistics
statIO = StatisticIO()
for site in options["sites"]:
siteData = projData.getSiteData(site)
measurements = siteData.getMeasurements()
for meas in measurements:
sampleFreq = siteData.getMeasurementSampleFreq(meas)
if sampleFreq not in options["sampleFreqs"]:
continue
projectText(
"Calculating stats for site {}, measurement {}".format(site, meas)
)
decParams = getDecimationParameters(sampleFreq, projData.config)
numLevels = decParams.numLevels
specReader = SpectrumReader(
os.path.join(siteData.getMeasurementSpecPath(meas), options["specdir"])
)
# calculate statistics for decimation level if spectra file exists
for declevel in range(0, numLevels):
check = specReader.openBinaryForReading("spectra", declevel)
if not check:
continue
refTime = specReader.getReferenceTime()
winSize = specReader.getWindowSize()
winOlap = specReader.getWindowOverlap()
numWindows = specReader.getNumWindows()
sampleFreqDec = specReader.getSampleFreq()
evalFreq = decParams.getEvalFrequenciesForLevel(declevel)
# dictionary for saving statistic data
statData = {}
for stat in options["stats"]:
statElements = getStatElements(stat)
statData[stat] = StatisticData(
stat, refTime, sampleFreqDec, winSize, winOlap
)
statData[stat].setStatParams(numWindows, statElements, evalFreq)
statData[stat].comments = specReader.getComments()
statData[stat].addComment(projData.config.getConfigComment())
statData[stat].addComment("Calculating statistic: {}".format(stat))
statData[stat].addComment(
"Statistic components: {}".format(listToString(statElements))
)
# get all the spectra data in batch and process all the windows
spectraData, globalIndices = specReader.readBinaryBatchGlobal()
if options["ncores"] > 0:
out = multiStatistics(
options["ncores"], spectraData, evalFreq, options["stats"]
)
for iW in range(numWindows):
for stat in options["stats"]:
statData[stat].addStat(iW, globalIndices[iW], out[iW][stat])
else:
statCalculator = StatisticCalculator()
for iW in range(numWindows):
winSpecData = spectraData[iW]
winStatData = calculateWindowStatistics(
winSpecData,
evalFreq,
options["stats"],
statCalculator=statCalculator,
)
for stat in options["stats"]:
statData[stat].addStat(
iW, globalIndices[iW], winStatData[stat]
)
specReader.closeFile()
# save statistic
for stat in options["stats"]:
statIO.setDatapath(
os.path.join(
siteData.getMeasurementStatPath(meas), options["specdir"]
)
)
statIO.write(statData[stat], declevel)