def test_local_regression() -> None:
"""Test selecting masked windows"""
from datapaths import path_integrated_singlesite, path_integrated_singlesite_config
from resistics.project.io import loadProject
from resistics.project.transfunc import getTransferFunctionData, viewImpedance
from resistics.project.shortcuts import (
getWindowSelector,
getDecimationParameters,
getWindowParameters,
)
from localtest2 import LocalRegressor
# from localtest import LocalRegressor
import numpy as np
proj = loadProject(path_integrated_singlesite,
str(path_integrated_singlesite_config))
sites = ["M7_4096"]
decParams = getDecimationParameters(4096, proj.config)
winParams = getWindowParameters(decParams, proj.config)
selector = getWindowSelector(
proj, decParams, winParams,
proj.config.configParams["Spectra"]["specdir"])
# now add a mask
selector.setSites(sites + sites)
selector.addWindowMask(sites[0], "coh_tf")
selector.calcSharedWindows()
# add the input and output site
processor = LocalRegressor(selector, "")
processor.setInput(sites[0], ["Hx", "Hy"])
processor.setOutput(sites[0], ["Ex", "Ey"])
processor.postpend = "stacked"
processor.printInfo()
processor.process(4)
def test_window_selector() -> None:
"""Test selecting masked windows"""
from datapaths import path_integrated_singlesite, path_integrated_singlesite_config
from resistics.project.io import loadProject
from resistics.project.shortcuts import (
getWindowSelector,
getDecimationParameters,
getWindowParameters,
)
import numpy as np
proj = loadProject(path_integrated_singlesite,
str(path_integrated_singlesite_config))
sites = ["M7_4096"]
# get decimation, window parameters and selector
decParams = getDecimationParameters(4096, proj.config)
winParams = getWindowParameters(decParams, proj.config)
selector = getWindowSelector(
proj, decParams, winParams,
proj.config.configParams["Spectra"]["specdir"])
# now add a mask
selector.setSites(sites)
selector.addWindowMask(sites[0], "coh_tf")
selector.calcSharedWindows()
declevel = 0
unmaskedWindows = selector.getUnmaskedWindowsLevel(declevel)
spectraBatches = selector.getSpecReaderBatches(declevel)
for batch in spectraBatches:
batch = spectraBatches[0]
reader = batch[sites[0]]
batchedWindows = unmaskedWindows.intersection(
set(range(batch["globalrange"][0], batch["globalrange"][1] + 1)))
# read the batch
batchData, batchGlobalIndices = reader.readBinaryBatchGlobal(
batchedWindows)
# for each window, check to make sure all correct
for testGlobalIndex in list(batchedWindows):
winData1 = reader.readBinaryWindowGlobal(testGlobalIndex)
# matching readBatch index
matchingBatchIndex = list(batchGlobalIndices).index(
testGlobalIndex)
winData2 = batchData[matchingBatchIndex]
# test winData1 and winData2
chans = winData1.chans
for chan in chans:
assert np.array_equal(winData1.data[chan], winData2.data[chan])
def test_statistic_transfunc() -> None:
"""Get the transfer functions statistic"""
from datapaths import path_integrated_singlesite, path_integrated_singlesite_config
from resistics.project.io import loadProject
from resistics.project.statistics import getStatisticData
# load project
sites = ["M7_4096"]
proj = loadProject(path_integrated_singlesite,
str(path_integrated_singlesite_config))
statData = getStatisticData(proj, sites[0], "meas_2016-02-25_02-00-00",
"transferFunction", 1)
data = statData.getStatLocal(0)
print(statData.winStats)
print(data)
# now do the same window through the
from resistics.project.transfunc import getTransferFunctionData
from resistics.project.shortcuts import (
getWindowSelector,
getDecimationParameters,
getWindowParameters,
)
from localtest3 import LocalRegressor
import numpy as np
proj = loadProject(path_integrated_singlesite,
str(path_integrated_singlesite_config))
sites = ["M7_4096"]
decParams = getDecimationParameters(4096, proj.config)
winParams = getWindowParameters(decParams, proj.config)
selector = getWindowSelector(
proj, decParams, winParams,
proj.config.configParams["Spectra"]["specdir"])
# now add a mask
selector.setSites(sites + sites)
selector.calcSharedWindows()
# add the input and output site
processor = LocalRegressor(selector, "")
processor.setInput(sites[0], ["Hx", "Hy"])
processor.setOutput(sites[0], ["Ex", "Ey"])
processor.postpend = "check"
processor.process(4)
from datapaths import projectPath, imagePath from resistics.project.io import loadProject # load project and configuration file projData = loadProject(projectPath, configFile="tutorialConfig.ini") # get decimation parameters from resistics.project.shortcuts import getDecimationParameters decimationParameters = getDecimationParameters(4096, projData.config) decimationParameters.printInfo() # get the window parameters from resistics.project.shortcuts import getWindowParameters windowParameters = getWindowParameters(decimationParameters, projData.config) windowParameters.printInfo() from resistics.project.shortcuts import getWindowSelector decimationParameters = getDecimationParameters(128, projData.config) windowParameters = getWindowParameters(decimationParameters, projData.config) selector = getWindowSelector(projData, decimationParameters, windowParameters) selector.printInfo() # add a site and print the information to the terminal selector.setSites(["site1"]) selector.printInfo() # calculate shared windows selector.calcSharedWindows()
def calculateMask(projData: ProjectData, maskData: MaskData, **kwargs):
"""Calculate masks sites
Parameters
----------
projData : projectData
A project instance
maskData : MaskData
A mask data instance
sites : List[str], optional
A list of sites to calculate masks for
specdir : str, optional
The spectra directory for which to calculate statistics
"""
options = {}
options["sites"] = projData.getSites()
options["specdir"] = projData.config.configParams["Spectra"]["specdir"]
options = parseKeywords(options, kwargs)
# create a maskCalculator object
maskCalc = MaskCalculator(projData, maskData, specdir=options["specdir"])
maskIO = MaskIO()
sampleFreq = maskData.sampleFreq
# loop over sites
for site in options["sites"]:
# see if there is a sample freq
siteData = projData.getSiteData(site)
siteSampleFreqs = siteData.getSampleFreqs()
if sampleFreq not in siteSampleFreqs:
continue
# decimation and window parameters
decParams = getDecimationParameters(sampleFreq, projData.config)
decParams.printInfo()
winParams = getWindowParameters(decParams, projData.config)
# clear previous windows from maskCalc
maskCalc.clearMaskWindows()
# calculate masked windows
maskCalc.applyConstraints(site)
maskCalc.maskData.printInfo()
# write maskIO file
maskIO.datapath = os.path.join(
siteData.getSpecdirMaskPath(options["specdir"]))
maskIO.write(maskCalc.maskData)
# test with the window selector
winSelector = WindowSelector(projData,
sampleFreq,
decParams,
winParams,
specdir=options["specdir"])
winSelector.setSites([site])
winSelector.addWindowMask(site, maskData.maskName)
winSelector.calcSharedWindows()
winSelector.printInfo()
winSelector.printDatetimeConstraints()
winSelector.printWindowMasks()
winSelector.printSharedWindows()
winSelector.printWindowsForFrequency()
def calculateSpectra(projData: ProjectData, **kwargs) -> None:
"""Calculate spectra for the project time data
The philosophy is that spectra are calculated out for all data and later limited using statistics and time constraints
Parameters
----------
projData : ProjectData
A project data object
sites : str, List[str], optional
Either a single site or a list of sites
sampleFreqs : int, float, List[float], optional
The frequencies in Hz for which to calculate the spectra. Either a single frequency or a list of them.
chans : List[str], optional
The channels for which to calculate out the spectra
polreverse : Dict[str, bool]
Keys are channels and values are boolean flags for reversing
scale : Dict[str, float]
Keys are channels and values are floats to multiply the channel data by
calibrate : bool, optional
Flag whether to calibrate the data or not
notch : List[float], optional
List of frequencies to notch
filter : Dict, optional
Filter parameters
specdir : str, optional
The spectra directory to save the spectra data in
ncores : int, optional
The number of cores to run the transfer function calculations on
"""
from resistics.spectra.io import SpectrumWriter
from resistics.decimate.decimator import Decimator
from resistics.window.windower import Windower
from resistics.project.shortcuts import (
getCalibrator,
getDecimationParameters,
getWindowParameters,
)
from resistics.project.preprocess import (
applyPolarisationReversalOptions,
applyScaleOptions,
applyCalibrationOptions,
applyFilterOptions,
applyNotchOptions,
)
options = {}
options["sites"] = projData.getSites()
options["sampleFreqs"]: List[float] = projData.getSampleFreqs()
options["chans"]: List[str] = []
options["polreverse"]: Union[bool, Dict[str, bool]] = False
options["scale"]: Union[bool, Dict[str, float]] = False
options["calibrate"]: bool = True
options["notch"]: List[float] = []
options["filter"]: Dict = {}
options["specdir"]: str = projData.config.configParams["Spectra"][
"specdir"]
options["ncores"] = projData.config.getSpectraCores()
options = parseKeywords(options, kwargs)
# prepare calibrator
cal = getCalibrator(projData.calPath, projData.config)
if options["calibrate"]:
cal.printInfo()
datetimeRef = projData.refTime
for site in options["sites"]:
siteData = projData.getSiteData(site)
siteData.printInfo()
# calculate spectra for each frequency
for sampleFreq in options["sampleFreqs"]:
measurements = siteData.getMeasurements(sampleFreq)
projectText(
"Site {} has {:d} measurement(s) at sampling frequency {:.2f}".
format(site, len(measurements), sampleFreq))
if len(measurements) == 0:
continue # no data files at this sample rate
for meas in measurements:
projectText(
"Calculating spectra for site {} and measurement {}".
format(site, meas))
# get measurement start and end times - this is the time of the first and last sample
reader = siteData.getMeasurement(meas)
startTime = siteData.getMeasurementStart(meas)
stopTime = siteData.getMeasurementEnd(meas)
dataChans = (options["chans"] if len(options["chans"]) > 0 else
reader.getChannels())
timeData = reader.getPhysicalData(startTime,
stopTime,
chans=dataChans)
timeData.addComment(breakComment())
timeData.addComment("Calculating project spectra")
timeData.addComment(projData.config.getConfigComment())
# apply various options
applyPolarisationReversalOptions(options, timeData)
applyScaleOptions(options, timeData)
applyCalibrationOptions(options, cal, timeData, reader)
applyFilterOptions(options, timeData)
applyNotchOptions(options, timeData)
# define decimation and window parameters
decParams = getDecimationParameters(sampleFreq,
projData.config)
numLevels = decParams.numLevels
winParams = getWindowParameters(decParams, projData.config)
dec = Decimator(timeData, decParams)
timeData.addComment(
"Decimating with {} levels and {} frequencies per level".
format(numLevels, decParams.freqPerLevel))
# loop through decimation levels
for declevel in range(0, numLevels):
# get the data for the current level
check = dec.incrementLevel()
if not check:
break # not enough data
timeData = dec.timeData
# create the windower and give it window parameters for current level
sampleFreqDec = dec.sampleFreq
win = Windower(
datetimeRef,
timeData,
winParams.getWindowSize(declevel),
winParams.getOverlap(declevel),
)
if win.numWindows < 2:
break # do no more decimation
# print information and add some comments
projectText(
"Calculating spectra for decimation level {}".format(
declevel))
timeData.addComment(
"Evaluation frequencies for this level {}".format(
listToString(
decParams.getEvalFrequenciesForLevel(
declevel))))
timeData.addComment(
"Windowing with window size {} samples and overlap {} samples"
.format(
winParams.getWindowSize(declevel),
winParams.getOverlap(declevel),
))
if projData.config.configParams["Spectra"]["applywindow"]:
timeData.addComment(
"Performing fourier transform with window function {}"
.format(projData.config.configParams["Spectra"]
["windowfunc"]))
else:
timeData.addComment(
"Performing fourier transform with no window function"
)
# collect time data
timeDataList = []
for iW in range(0, win.numWindows):
timeDataList.append(win.getData(iW))
# open spectra file for saving
specPath = os.path.join(
siteData.getMeasurementSpecPath(meas),
options["specdir"])
specWrite = SpectrumWriter(specPath, datetimeRef)
specWrite.openBinaryForWriting(
"spectra",
declevel,
sampleFreqDec,
winParams.getWindowSize(declevel),
winParams.getOverlap(declevel),
win.winOffset,
win.numWindows,
dataChans,
)
if options["ncores"] > 0:
specDataList = multiSpectra(
options["ncores"],
timeDataList,
sampleFreqDec,
winParams.getWindowSize(declevel),
projData.config.configParams,
)
else:
specDataList = calculateWindowSpectra(
timeDataList,
sampleFreqDec,
winParams.getWindowSize(declevel),
projData.config.configParams,
)
# write out to spectra file
for iW in range(0, win.numWindows):
specWrite.writeBinary(specDataList[iW])
specWrite.writeCommentsFile(timeData.getComments())
specWrite.closeFile()
def calculateRemoteStatistics(projData: ProjectData, remoteSite: str, **kwargs):
"""Calculate statistics involving a remote reference site
Parameters
----------
projData : ProjectData
A project data instance
remoteSite : str
The name of the site to use as the remote site
sites : List[str], optional
A list of sites to calculate statistics for
sampleFreqs : List[float], optional
List of sampling frequencies for which to calculate statistics
specdir : str, optional
The spectra directory for which to calculate statistics
remotestats : List[str], optional
The statistics to calculate out. Acceptable statistics are: "RR_coherence", "RR_coherenceEqn", "RR_absvalEqn", "RR_transferFunction", "RR_resPhase". Configuration file values are used by default.
"""
from resistics.statistics.io import StatisticIO
from resistics.statistics.calculator import StatisticCalculator
from resistics.project.shortcuts import (
getDecimationParameters,
getWindowParameters,
getWindowSelector,
)
options = {}
options["sites"] = projData.getSites()
options["sampleFreqs"] = projData.getSampleFreqs()
options["chans"] = []
options["specdir"] = projData.config.configParams["Spectra"]["specdir"]
options["remotestats"] = projData.config.configParams["Statistics"]["remotestats"]
options["ncores"] = projData.config.getStatisticCores()
options = parseKeywords(options, kwargs)
projectText(
"Calculating stats: {} for sites: {} with remote site {}".format(
listToString(options["remotestats"]),
listToString(options["sites"]),
remoteSite,
)
)
statIO = StatisticIO()
for site in options["sites"]:
siteData = projData.getSiteData(site)
measurements = siteData.getMeasurements()
for meas in measurements:
sampleFreq = siteData.getMeasurementSampleFreq(meas)
if sampleFreq not in options["sampleFreqs"]:
continue
projectText(
"Calculating stats for site {}, measurement {} with reference {}".format(
site, meas, remoteSite
)
)
# decimation and window parameters
decParams = getDecimationParameters(sampleFreq, projData.config)
numLevels = decParams.numLevels
winParams = getWindowParameters(decParams, projData.config)
# create the window selector and find the shared windows
winSelector = getWindowSelector(projData, decParams, winParams)
winSelector.setSites([site, remoteSite])
winSelector.calcSharedWindows()
# create the spectrum reader
specReader = SpectrumReader(
os.path.join(siteData.getMeasurementSpecPath(meas), options["specdir"])
)
# calculate statistics for decimation level if spectra file exists
for declevel in range(0, numLevels):
check = specReader.openBinaryForReading("spectra", declevel)
if not check:
continue
# information regarding only this spectra file
refTime = specReader.getReferenceTime()
winSize = specReader.getWindowSize()
winOlap = specReader.getWindowOverlap()
numWindows = specReader.getNumWindows()
evalFreq = decParams.getEvalFrequenciesForLevel(declevel)
sampleFreqDec = specReader.getSampleFreq()
globalOffset = specReader.getGlobalOffset()
# find size of the intersection between the windows in this spectra file and the shared windows
sharedWindows = winSelector.getSharedWindowsLevel(declevel)
sharedWindowsMeas = sharedWindows.intersection(
set(np.arange(globalOffset, globalOffset + numWindows))
)
sharedWindowsMeas = sorted(list(sharedWindowsMeas))
numSharedWindows = len(sharedWindowsMeas)
statData = {}
# create the statistic handlers
for stat in options["remotestats"]:
statElements = getStatElements(stat)
statData[stat] = StatisticData(
stat, refTime, sampleFreqDec, winSize, winOlap
)
# with remote reference the number of windows is number of shared windows
statData[stat].setStatParams(
numSharedWindows, statElements, evalFreq
)
statData[stat].comments = specReader.getComments()
statData[stat].addComment(projData.config.getConfigComment())
statData[stat].addComment(
"Calculating remote statistic: {}".format(stat)
)
statData[stat].addComment(
"Statistic components: {}".format(listToString(statElements))
)
# collect the spectra data
spectraData, _globalIndices = specReader.readBinaryBatchGlobal(
sharedWindowsMeas
)
remoteData = []
for globalWindow in sharedWindowsMeas:
_, remoteReader = winSelector.getSpecReaderForWindow(
remoteSite, declevel, globalWindow
)
remoteData.append(remoteReader.readBinaryWindowGlobal(globalWindow))
# calculate
if options["ncores"] > 0:
out = multiStatistics(
options["ncores"],
spectraData,
evalFreq,
options["remotestats"],
remoteData=remoteData,
)
for iW, globalWindow in enumerate(sharedWindowsMeas):
for stat in options["remotestats"]:
statData[stat].addStat(iW, globalWindow, out[iW][stat])
else:
statCalculator = StatisticCalculator()
for iW, globalWindow in enumerate(sharedWindowsMeas):
winStatData = calculateWindowStatistics(
spectraData[iW],
evalFreq,
options["remotestats"],
remoteSpecData=remoteData[iW],
statCalculator=statCalculator,
)
for stat in options["remotestats"]:
statData[stat].addStat(iW, globalWindow, winStatData[stat])
# save statistic
for stat in options["remotestats"]:
statIO.setDatapath(
os.path.join(
siteData.getMeasurementStatPath(meas), options["specdir"]
)
)
statIO.write(statData[stat], declevel)
def processSite(projData: ProjectData, site: str,
sampleFreq: Union[int, float], **kwargs):
"""Process a single sampling frequency for a site
The site passed is assumed to be the output site (the output channels will come from this site). If channels from a different site are desired to be used as the input channels, this can be done by specifying the optional inputsite argument.
.. todo::
Give a few different examples here
Parameters
----------
projData : ProjectData
The project data instance for the project
site : str
Site to process
sampleFreq : float, int
Sample frequency to process
specdir : str, optional
The spectra directories to use
inchans : List[str], optional
Channels to use as the input of the linear system
inputsite : str, optional
Site from which to take the input channels. The default is to use input and output channels from the same site
outchans : List[str], optional
Channels to use as the output of the linear system
remotesite : str, optional
The site to use as the remote site
remotechans : List[str], optional
Channels to use from the remote reference site
crosschannels : List[str], optional
List of channels to use for cross powers
masks : Dict, optional
Masks dictionary for passing mask data. The key should be a site name and the value should either be a string for a single mask or a list of multiple masks.
datetimes : List, optional
List of datetime constraints, each one as a dictionary. For example [{"type": "datetime", "start": 2018-08-08 00:00:00, "end": 2018-08-08 16:00:00, "levels": [0,1]}]. Note that levels is optional.
postpend : str, optional
String to postpend to the transfer function output
ncores : int, optional
The number of cores to run the transfer function calculations on
"""
from resistics.decimate.decimator import Decimator
from resistics.window.selector import WindowSelector
from resistics.project.shortcuts import (
getDecimationParameters,
getWindowParameters,
getWindowSelector,
getLocalRegressor,
getRemoteRegressor,
)
options = {}
options["specdir"] = projData.config.configParams["Spectra"]["specdir"]
options["inchans"] = ["Hx", "Hy"]
options["inputsite"] = ""
options["outchans"] = ["Ex", "Ey"]
options["remotesite"] = ""
options["remotechans"] = options["inchans"]
options["crosschannels"] = []
options["masks"] = {}
options["datetimes"] = []
options["postpend"] = ""
options["ncores"] = projData.config.getSolverCores()
options = parseKeywords(options, kwargs)
if options["inputsite"] == "":
options["inputsite"] = site
projectText("Processing site {}, sampling frequency {}".format(
site, sampleFreq))
siteData = projData.getSiteData(site)
# define decimation parameters
decParams = getDecimationParameters(sampleFreq, projData.config)
decParams.printInfo()
winParams = getWindowParameters(decParams, projData.config)
# window selector
winSelector = getWindowSelector(projData, decParams, winParams,
options["specdir"])
# if two sites are duplicated (e.g. input site and output site), winSelector only uses distinct sites. Hence using site and inputSite is no problem even if they are the same
processSites = []
if options["remotesite"]:
processSites = [site, options["inputsite"], options["remotesite"]]
winSelector.setSites(processSites)
else:
# if no remote site, then single site processing
processSites = [site, options["inputsite"]]
winSelector.setSites(processSites)
# add window masks
if len(list(options["masks"].keys())) > 0:
for maskSite in options["masks"]:
if maskSite not in processSites:
# there is a site in the masks dictionary which is of no interest
continue
if isinstance(options["masks"][maskSite], str):
# a single mask
winSelector.addWindowMask(maskSite, options["masks"][maskSite])
continue
if all(
isinstance(item, str)
for item in options["masks"][maskSite]):
# list of masks for the site
for mask in options["masks"][maskSite]:
winSelector.addWindowMask(maskSite, mask)
# add datetime constraints
for dC in options["datetimes"]:
levels = None
if "levels" in dC:
levels = dC["levels"]
if dC["type"] == "datetime":
winSelector.addDatetimeConstraint(dC["start"], dC["stop"], levels)
if dC["type"] == "time":
winSelector.addTimeConstraint(dC["start"], dC["stop"], levels)
if dC["type"] == "date":
winSelector.addDateConstraint(dC["date"], levels)
# calculate the shared windows and print info
winSelector.calcSharedWindows()
winSelector.printInfo()
winSelector.printDatetimeConstraints()
winSelector.printWindowMasks()
winSelector.printSharedWindows()
winSelector.printWindowsForFrequency()
# now have the windows, pass the winSelector to processors
outPath = siteData.transFuncPath
if options["remotesite"]:
projectText(
"Remote reference processing with sites: in = {}, out = {}, reference = {}"
.format(options["inputsite"], site, options["remotesite"]))
processor = getRemoteRegressor(winSelector, outPath, projData.config)
processor.setRemote(options["remotesite"], options["remotechans"])
else:
projectText(
"Single site processing with sites: in = {}, out = {}".format(
options["inputsite"], site))
processor = getLocalRegressor(winSelector, outPath, projData.config)
# add the input and output site
processor.setCores(options["ncores"])
processor.setInput(options["inputsite"], options["inchans"])
processor.setOutput(site, options["outchans"])
if len(options["crosschannels"]) > 0:
processor.crossChannels = options["crosschannels"]
processor.postpend = options["postpend"]
processor.printInfo()
projectText("Processing data using {} cores".format(options["ncores"]))
processor.process()