def test_local_regression() -> None: """Test selecting masked windows""" from datapaths import path_integrated_singlesite, path_integrated_singlesite_config from resistics.project.io import loadProject from resistics.project.transfunc import getTransferFunctionData, viewImpedance from resistics.project.shortcuts import ( getWindowSelector, getDecimationParameters, getWindowParameters, ) from localtest2 import LocalRegressor # from localtest import LocalRegressor import numpy as np proj = loadProject(path_integrated_singlesite, str(path_integrated_singlesite_config)) sites = ["M7_4096"] decParams = getDecimationParameters(4096, proj.config) winParams = getWindowParameters(decParams, proj.config) selector = getWindowSelector( proj, decParams, winParams, proj.config.configParams["Spectra"]["specdir"]) # now add a mask selector.setSites(sites + sites) selector.addWindowMask(sites[0], "coh_tf") selector.calcSharedWindows() # add the input and output site processor = LocalRegressor(selector, "") processor.setInput(sites[0], ["Hx", "Hy"]) processor.setOutput(sites[0], ["Ex", "Ey"]) processor.postpend = "stacked" processor.printInfo() processor.process(4)
def test_window_selector() -> None: """Test selecting masked windows""" from datapaths import path_integrated_singlesite, path_integrated_singlesite_config from resistics.project.io import loadProject from resistics.project.shortcuts import ( getWindowSelector, getDecimationParameters, getWindowParameters, ) import numpy as np proj = loadProject(path_integrated_singlesite, str(path_integrated_singlesite_config)) sites = ["M7_4096"] # get decimation, window parameters and selector decParams = getDecimationParameters(4096, proj.config) winParams = getWindowParameters(decParams, proj.config) selector = getWindowSelector( proj, decParams, winParams, proj.config.configParams["Spectra"]["specdir"]) # now add a mask selector.setSites(sites) selector.addWindowMask(sites[0], "coh_tf") selector.calcSharedWindows() declevel = 0 unmaskedWindows = selector.getUnmaskedWindowsLevel(declevel) spectraBatches = selector.getSpecReaderBatches(declevel) for batch in spectraBatches: batch = spectraBatches[0] reader = batch[sites[0]] batchedWindows = unmaskedWindows.intersection( set(range(batch["globalrange"][0], batch["globalrange"][1] + 1))) # read the batch batchData, batchGlobalIndices = reader.readBinaryBatchGlobal( batchedWindows) # for each window, check to make sure all correct for testGlobalIndex in list(batchedWindows): winData1 = reader.readBinaryWindowGlobal(testGlobalIndex) # matching readBatch index matchingBatchIndex = list(batchGlobalIndices).index( testGlobalIndex) winData2 = batchData[matchingBatchIndex] # test winData1 and winData2 chans = winData1.chans for chan in chans: assert np.array_equal(winData1.data[chan], winData2.data[chan])
def test_statistic_transfunc() -> None: """Get the transfer functions statistic""" from datapaths import path_integrated_singlesite, path_integrated_singlesite_config from resistics.project.io import loadProject from resistics.project.statistics import getStatisticData # load project sites = ["M7_4096"] proj = loadProject(path_integrated_singlesite, str(path_integrated_singlesite_config)) statData = getStatisticData(proj, sites[0], "meas_2016-02-25_02-00-00", "transferFunction", 1) data = statData.getStatLocal(0) print(statData.winStats) print(data) # now do the same window through the from resistics.project.transfunc import getTransferFunctionData from resistics.project.shortcuts import ( getWindowSelector, getDecimationParameters, getWindowParameters, ) from localtest3 import LocalRegressor import numpy as np proj = loadProject(path_integrated_singlesite, str(path_integrated_singlesite_config)) sites = ["M7_4096"] decParams = getDecimationParameters(4096, proj.config) winParams = getWindowParameters(decParams, proj.config) selector = getWindowSelector( proj, decParams, winParams, proj.config.configParams["Spectra"]["specdir"]) # now add a mask selector.setSites(sites + sites) selector.calcSharedWindows() # add the input and output site processor = LocalRegressor(selector, "") processor.setInput(sites[0], ["Hx", "Hy"]) processor.setOutput(sites[0], ["Ex", "Ey"]) processor.postpend = "check" processor.process(4)
from datapaths import projectPath, imagePath from resistics.project.io import loadProject # load project and configuration file projData = loadProject(projectPath, configFile="tutorialConfig.ini") # get decimation parameters from resistics.project.shortcuts import getDecimationParameters decimationParameters = getDecimationParameters(4096, projData.config) decimationParameters.printInfo() # get the window parameters from resistics.project.shortcuts import getWindowParameters windowParameters = getWindowParameters(decimationParameters, projData.config) windowParameters.printInfo() from resistics.project.shortcuts import getWindowSelector decimationParameters = getDecimationParameters(128, projData.config) windowParameters = getWindowParameters(decimationParameters, projData.config) selector = getWindowSelector(projData, decimationParameters, windowParameters) selector.printInfo() # add a site and print the information to the terminal selector.setSites(["site1"]) selector.printInfo() # calculate shared windows selector.calcSharedWindows()
def calculateMask(projData: ProjectData, maskData: MaskData, **kwargs): """Calculate masks sites Parameters ---------- projData : projectData A project instance maskData : MaskData A mask data instance sites : List[str], optional A list of sites to calculate masks for specdir : str, optional The spectra directory for which to calculate statistics """ options = {} options["sites"] = projData.getSites() options["specdir"] = projData.config.configParams["Spectra"]["specdir"] options = parseKeywords(options, kwargs) # create a maskCalculator object maskCalc = MaskCalculator(projData, maskData, specdir=options["specdir"]) maskIO = MaskIO() sampleFreq = maskData.sampleFreq # loop over sites for site in options["sites"]: # see if there is a sample freq siteData = projData.getSiteData(site) siteSampleFreqs = siteData.getSampleFreqs() if sampleFreq not in siteSampleFreqs: continue # decimation and window parameters decParams = getDecimationParameters(sampleFreq, projData.config) decParams.printInfo() winParams = getWindowParameters(decParams, projData.config) # clear previous windows from maskCalc maskCalc.clearMaskWindows() # calculate masked windows maskCalc.applyConstraints(site) maskCalc.maskData.printInfo() # write maskIO file maskIO.datapath = os.path.join( siteData.getSpecdirMaskPath(options["specdir"])) maskIO.write(maskCalc.maskData) # test with the window selector winSelector = WindowSelector(projData, sampleFreq, decParams, winParams, specdir=options["specdir"]) winSelector.setSites([site]) winSelector.addWindowMask(site, maskData.maskName) winSelector.calcSharedWindows() winSelector.printInfo() winSelector.printDatetimeConstraints() winSelector.printWindowMasks() winSelector.printSharedWindows() winSelector.printWindowsForFrequency()
def calculateSpectra(projData: ProjectData, **kwargs) -> None: """Calculate spectra for the project time data The philosophy is that spectra are calculated out for all data and later limited using statistics and time constraints Parameters ---------- projData : ProjectData A project data object sites : str, List[str], optional Either a single site or a list of sites sampleFreqs : int, float, List[float], optional The frequencies in Hz for which to calculate the spectra. Either a single frequency or a list of them. chans : List[str], optional The channels for which to calculate out the spectra polreverse : Dict[str, bool] Keys are channels and values are boolean flags for reversing scale : Dict[str, float] Keys are channels and values are floats to multiply the channel data by calibrate : bool, optional Flag whether to calibrate the data or not notch : List[float], optional List of frequencies to notch filter : Dict, optional Filter parameters specdir : str, optional The spectra directory to save the spectra data in ncores : int, optional The number of cores to run the transfer function calculations on """ from resistics.spectra.io import SpectrumWriter from resistics.decimate.decimator import Decimator from resistics.window.windower import Windower from resistics.project.shortcuts import ( getCalibrator, getDecimationParameters, getWindowParameters, ) from resistics.project.preprocess import ( applyPolarisationReversalOptions, applyScaleOptions, applyCalibrationOptions, applyFilterOptions, applyNotchOptions, ) options = {} options["sites"] = projData.getSites() options["sampleFreqs"]: List[float] = projData.getSampleFreqs() options["chans"]: List[str] = [] options["polreverse"]: Union[bool, Dict[str, bool]] = False options["scale"]: Union[bool, Dict[str, float]] = False options["calibrate"]: bool = True options["notch"]: List[float] = [] options["filter"]: Dict = {} options["specdir"]: str = projData.config.configParams["Spectra"][ "specdir"] options["ncores"] = projData.config.getSpectraCores() options = parseKeywords(options, kwargs) # prepare calibrator cal = getCalibrator(projData.calPath, projData.config) if options["calibrate"]: cal.printInfo() datetimeRef = projData.refTime for site in options["sites"]: siteData = projData.getSiteData(site) siteData.printInfo() # calculate spectra for each frequency for sampleFreq in options["sampleFreqs"]: measurements = siteData.getMeasurements(sampleFreq) projectText( "Site {} has {:d} measurement(s) at sampling frequency {:.2f}". format(site, len(measurements), sampleFreq)) if len(measurements) == 0: continue # no data files at this sample rate for meas in measurements: projectText( "Calculating spectra for site {} and measurement {}". format(site, meas)) # get measurement start and end times - this is the time of the first and last sample reader = siteData.getMeasurement(meas) startTime = siteData.getMeasurementStart(meas) stopTime = siteData.getMeasurementEnd(meas) dataChans = (options["chans"] if len(options["chans"]) > 0 else reader.getChannels()) timeData = reader.getPhysicalData(startTime, stopTime, chans=dataChans) timeData.addComment(breakComment()) timeData.addComment("Calculating project spectra") timeData.addComment(projData.config.getConfigComment()) # apply various options applyPolarisationReversalOptions(options, timeData) applyScaleOptions(options, timeData) applyCalibrationOptions(options, cal, timeData, reader) applyFilterOptions(options, timeData) applyNotchOptions(options, timeData) # define decimation and window parameters decParams = getDecimationParameters(sampleFreq, projData.config) numLevels = decParams.numLevels winParams = getWindowParameters(decParams, projData.config) dec = Decimator(timeData, decParams) timeData.addComment( "Decimating with {} levels and {} frequencies per level". format(numLevels, decParams.freqPerLevel)) # loop through decimation levels for declevel in range(0, numLevels): # get the data for the current level check = dec.incrementLevel() if not check: break # not enough data timeData = dec.timeData # create the windower and give it window parameters for current level sampleFreqDec = dec.sampleFreq win = Windower( datetimeRef, timeData, winParams.getWindowSize(declevel), winParams.getOverlap(declevel), ) if win.numWindows < 2: break # do no more decimation # print information and add some comments projectText( "Calculating spectra for decimation level {}".format( declevel)) timeData.addComment( "Evaluation frequencies for this level {}".format( listToString( decParams.getEvalFrequenciesForLevel( declevel)))) timeData.addComment( "Windowing with window size {} samples and overlap {} samples" .format( winParams.getWindowSize(declevel), winParams.getOverlap(declevel), )) if projData.config.configParams["Spectra"]["applywindow"]: timeData.addComment( "Performing fourier transform with window function {}" .format(projData.config.configParams["Spectra"] ["windowfunc"])) else: timeData.addComment( "Performing fourier transform with no window function" ) # collect time data timeDataList = [] for iW in range(0, win.numWindows): timeDataList.append(win.getData(iW)) # open spectra file for saving specPath = os.path.join( siteData.getMeasurementSpecPath(meas), options["specdir"]) specWrite = SpectrumWriter(specPath, datetimeRef) specWrite.openBinaryForWriting( "spectra", declevel, sampleFreqDec, winParams.getWindowSize(declevel), winParams.getOverlap(declevel), win.winOffset, win.numWindows, dataChans, ) if options["ncores"] > 0: specDataList = multiSpectra( options["ncores"], timeDataList, sampleFreqDec, winParams.getWindowSize(declevel), projData.config.configParams, ) else: specDataList = calculateWindowSpectra( timeDataList, sampleFreqDec, winParams.getWindowSize(declevel), projData.config.configParams, ) # write out to spectra file for iW in range(0, win.numWindows): specWrite.writeBinary(specDataList[iW]) specWrite.writeCommentsFile(timeData.getComments()) specWrite.closeFile()
def calculateRemoteStatistics(projData: ProjectData, remoteSite: str, **kwargs): """Calculate statistics involving a remote reference site Parameters ---------- projData : ProjectData A project data instance remoteSite : str The name of the site to use as the remote site sites : List[str], optional A list of sites to calculate statistics for sampleFreqs : List[float], optional List of sampling frequencies for which to calculate statistics specdir : str, optional The spectra directory for which to calculate statistics remotestats : List[str], optional The statistics to calculate out. Acceptable statistics are: "RR_coherence", "RR_coherenceEqn", "RR_absvalEqn", "RR_transferFunction", "RR_resPhase". Configuration file values are used by default. """ from resistics.statistics.io import StatisticIO from resistics.statistics.calculator import StatisticCalculator from resistics.project.shortcuts import ( getDecimationParameters, getWindowParameters, getWindowSelector, ) options = {} options["sites"] = projData.getSites() options["sampleFreqs"] = projData.getSampleFreqs() options["chans"] = [] options["specdir"] = projData.config.configParams["Spectra"]["specdir"] options["remotestats"] = projData.config.configParams["Statistics"]["remotestats"] options["ncores"] = projData.config.getStatisticCores() options = parseKeywords(options, kwargs) projectText( "Calculating stats: {} for sites: {} with remote site {}".format( listToString(options["remotestats"]), listToString(options["sites"]), remoteSite, ) ) statIO = StatisticIO() for site in options["sites"]: siteData = projData.getSiteData(site) measurements = siteData.getMeasurements() for meas in measurements: sampleFreq = siteData.getMeasurementSampleFreq(meas) if sampleFreq not in options["sampleFreqs"]: continue projectText( "Calculating stats for site {}, measurement {} with reference {}".format( site, meas, remoteSite ) ) # decimation and window parameters decParams = getDecimationParameters(sampleFreq, projData.config) numLevels = decParams.numLevels winParams = getWindowParameters(decParams, projData.config) # create the window selector and find the shared windows winSelector = getWindowSelector(projData, decParams, winParams) winSelector.setSites([site, remoteSite]) winSelector.calcSharedWindows() # create the spectrum reader specReader = SpectrumReader( os.path.join(siteData.getMeasurementSpecPath(meas), options["specdir"]) ) # calculate statistics for decimation level if spectra file exists for declevel in range(0, numLevels): check = specReader.openBinaryForReading("spectra", declevel) if not check: continue # information regarding only this spectra file refTime = specReader.getReferenceTime() winSize = specReader.getWindowSize() winOlap = specReader.getWindowOverlap() numWindows = specReader.getNumWindows() evalFreq = decParams.getEvalFrequenciesForLevel(declevel) sampleFreqDec = specReader.getSampleFreq() globalOffset = specReader.getGlobalOffset() # find size of the intersection between the windows in this spectra file and the shared windows sharedWindows = winSelector.getSharedWindowsLevel(declevel) sharedWindowsMeas = sharedWindows.intersection( set(np.arange(globalOffset, globalOffset + numWindows)) ) sharedWindowsMeas = sorted(list(sharedWindowsMeas)) numSharedWindows = len(sharedWindowsMeas) statData = {} # create the statistic handlers for stat in options["remotestats"]: statElements = getStatElements(stat) statData[stat] = StatisticData( stat, refTime, sampleFreqDec, winSize, winOlap ) # with remote reference the number of windows is number of shared windows statData[stat].setStatParams( numSharedWindows, statElements, evalFreq ) statData[stat].comments = specReader.getComments() statData[stat].addComment(projData.config.getConfigComment()) statData[stat].addComment( "Calculating remote statistic: {}".format(stat) ) statData[stat].addComment( "Statistic components: {}".format(listToString(statElements)) ) # collect the spectra data spectraData, _globalIndices = specReader.readBinaryBatchGlobal( sharedWindowsMeas ) remoteData = [] for globalWindow in sharedWindowsMeas: _, remoteReader = winSelector.getSpecReaderForWindow( remoteSite, declevel, globalWindow ) remoteData.append(remoteReader.readBinaryWindowGlobal(globalWindow)) # calculate if options["ncores"] > 0: out = multiStatistics( options["ncores"], spectraData, evalFreq, options["remotestats"], remoteData=remoteData, ) for iW, globalWindow in enumerate(sharedWindowsMeas): for stat in options["remotestats"]: statData[stat].addStat(iW, globalWindow, out[iW][stat]) else: statCalculator = StatisticCalculator() for iW, globalWindow in enumerate(sharedWindowsMeas): winStatData = calculateWindowStatistics( spectraData[iW], evalFreq, options["remotestats"], remoteSpecData=remoteData[iW], statCalculator=statCalculator, ) for stat in options["remotestats"]: statData[stat].addStat(iW, globalWindow, winStatData[stat]) # save statistic for stat in options["remotestats"]: statIO.setDatapath( os.path.join( siteData.getMeasurementStatPath(meas), options["specdir"] ) ) statIO.write(statData[stat], declevel)
def processSite(projData: ProjectData, site: str, sampleFreq: Union[int, float], **kwargs): """Process a single sampling frequency for a site The site passed is assumed to be the output site (the output channels will come from this site). If channels from a different site are desired to be used as the input channels, this can be done by specifying the optional inputsite argument. .. todo:: Give a few different examples here Parameters ---------- projData : ProjectData The project data instance for the project site : str Site to process sampleFreq : float, int Sample frequency to process specdir : str, optional The spectra directories to use inchans : List[str], optional Channels to use as the input of the linear system inputsite : str, optional Site from which to take the input channels. The default is to use input and output channels from the same site outchans : List[str], optional Channels to use as the output of the linear system remotesite : str, optional The site to use as the remote site remotechans : List[str], optional Channels to use from the remote reference site crosschannels : List[str], optional List of channels to use for cross powers masks : Dict, optional Masks dictionary for passing mask data. The key should be a site name and the value should either be a string for a single mask or a list of multiple masks. datetimes : List, optional List of datetime constraints, each one as a dictionary. For example [{"type": "datetime", "start": 2018-08-08 00:00:00, "end": 2018-08-08 16:00:00, "levels": [0,1]}]. Note that levels is optional. postpend : str, optional String to postpend to the transfer function output ncores : int, optional The number of cores to run the transfer function calculations on """ from resistics.decimate.decimator import Decimator from resistics.window.selector import WindowSelector from resistics.project.shortcuts import ( getDecimationParameters, getWindowParameters, getWindowSelector, getLocalRegressor, getRemoteRegressor, ) options = {} options["specdir"] = projData.config.configParams["Spectra"]["specdir"] options["inchans"] = ["Hx", "Hy"] options["inputsite"] = "" options["outchans"] = ["Ex", "Ey"] options["remotesite"] = "" options["remotechans"] = options["inchans"] options["crosschannels"] = [] options["masks"] = {} options["datetimes"] = [] options["postpend"] = "" options["ncores"] = projData.config.getSolverCores() options = parseKeywords(options, kwargs) if options["inputsite"] == "": options["inputsite"] = site projectText("Processing site {}, sampling frequency {}".format( site, sampleFreq)) siteData = projData.getSiteData(site) # define decimation parameters decParams = getDecimationParameters(sampleFreq, projData.config) decParams.printInfo() winParams = getWindowParameters(decParams, projData.config) # window selector winSelector = getWindowSelector(projData, decParams, winParams, options["specdir"]) # if two sites are duplicated (e.g. input site and output site), winSelector only uses distinct sites. Hence using site and inputSite is no problem even if they are the same processSites = [] if options["remotesite"]: processSites = [site, options["inputsite"], options["remotesite"]] winSelector.setSites(processSites) else: # if no remote site, then single site processing processSites = [site, options["inputsite"]] winSelector.setSites(processSites) # add window masks if len(list(options["masks"].keys())) > 0: for maskSite in options["masks"]: if maskSite not in processSites: # there is a site in the masks dictionary which is of no interest continue if isinstance(options["masks"][maskSite], str): # a single mask winSelector.addWindowMask(maskSite, options["masks"][maskSite]) continue if all( isinstance(item, str) for item in options["masks"][maskSite]): # list of masks for the site for mask in options["masks"][maskSite]: winSelector.addWindowMask(maskSite, mask) # add datetime constraints for dC in options["datetimes"]: levels = None if "levels" in dC: levels = dC["levels"] if dC["type"] == "datetime": winSelector.addDatetimeConstraint(dC["start"], dC["stop"], levels) if dC["type"] == "time": winSelector.addTimeConstraint(dC["start"], dC["stop"], levels) if dC["type"] == "date": winSelector.addDateConstraint(dC["date"], levels) # calculate the shared windows and print info winSelector.calcSharedWindows() winSelector.printInfo() winSelector.printDatetimeConstraints() winSelector.printWindowMasks() winSelector.printSharedWindows() winSelector.printWindowsForFrequency() # now have the windows, pass the winSelector to processors outPath = siteData.transFuncPath if options["remotesite"]: projectText( "Remote reference processing with sites: in = {}, out = {}, reference = {}" .format(options["inputsite"], site, options["remotesite"])) processor = getRemoteRegressor(winSelector, outPath, projData.config) processor.setRemote(options["remotesite"], options["remotechans"]) else: projectText( "Single site processing with sites: in = {}, out = {}".format( options["inputsite"], site)) processor = getLocalRegressor(winSelector, outPath, projData.config) # add the input and output site processor.setCores(options["ncores"]) processor.setInput(options["inputsite"], options["inchans"]) processor.setOutput(site, options["outchans"]) if len(options["crosschannels"]) > 0: processor.crossChannels = options["crosschannels"] processor.postpend = options["postpend"] processor.printInfo() projectText("Processing data using {} cores".format(options["ncores"])) processor.process()