def test_local_regression() -> None: """Test selecting masked windows""" from datapaths import path_integrated_singlesite, path_integrated_singlesite_config from resistics.project.io import loadProject from resistics.project.transfunc import getTransferFunctionData, viewImpedance from resistics.project.shortcuts import ( getWindowSelector, getDecimationParameters, getWindowParameters, ) from localtest2 import LocalRegressor # from localtest import LocalRegressor import numpy as np proj = loadProject(path_integrated_singlesite, str(path_integrated_singlesite_config)) sites = ["M7_4096"] decParams = getDecimationParameters(4096, proj.config) winParams = getWindowParameters(decParams, proj.config) selector = getWindowSelector( proj, decParams, winParams, proj.config.configParams["Spectra"]["specdir"]) # now add a mask selector.setSites(sites + sites) selector.addWindowMask(sites[0], "coh_tf") selector.calcSharedWindows() # add the input and output site processor = LocalRegressor(selector, "") processor.setInput(sites[0], ["Hx", "Hy"]) processor.setOutput(sites[0], ["Ex", "Ey"]) processor.postpend = "stacked" processor.printInfo() processor.process(4)
def test_statistic_transfunc() -> None: """Get the transfer functions statistic""" from datapaths import path_integrated_singlesite, path_integrated_singlesite_config from resistics.project.io import loadProject from resistics.project.statistics import getStatisticData # load project sites = ["M7_4096"] proj = loadProject(path_integrated_singlesite, str(path_integrated_singlesite_config)) statData = getStatisticData(proj, sites[0], "meas_2016-02-25_02-00-00", "transferFunction", 1) data = statData.getStatLocal(0) print(statData.winStats) print(data) # now do the same window through the from resistics.project.transfunc import getTransferFunctionData from resistics.project.shortcuts import ( getWindowSelector, getDecimationParameters, getWindowParameters, ) from localtest3 import LocalRegressor import numpy as np proj = loadProject(path_integrated_singlesite, str(path_integrated_singlesite_config)) sites = ["M7_4096"] decParams = getDecimationParameters(4096, proj.config) winParams = getWindowParameters(decParams, proj.config) selector = getWindowSelector( proj, decParams, winParams, proj.config.configParams["Spectra"]["specdir"]) # now add a mask selector.setSites(sites + sites) selector.calcSharedWindows() # add the input and output site processor = LocalRegressor(selector, "") processor.setInput(sites[0], ["Hx", "Hy"]) processor.setOutput(sites[0], ["Ex", "Ey"]) processor.postpend = "check" processor.process(4)
def test_loadProject() -> None: """Test loading the project""" from datapaths import path_integrated_singlesite from resistics.project.io import loadProject proj = loadProject(path_integrated_singlesite) assert proj.refTime.strftime("%Y-%m-%d %H:%M:%S.%f") == "2016-02-18 12:00:00.000000" assert ( proj.projStart.strftime("%Y-%m-%d %H:%M:%S.%f") == "2016-02-19 11:22:57.000000" ) assert proj.projEnd.strftime("%Y-%m-%d %H:%M:%S.%f") == "2016-02-25 02:29:59.999756" assert proj.sites == ["M7_4096"]
def test_window_selector() -> None: """Test selecting masked windows""" from datapaths import path_integrated_singlesite, path_integrated_singlesite_config from resistics.project.io import loadProject from resistics.project.shortcuts import ( getWindowSelector, getDecimationParameters, getWindowParameters, ) import numpy as np proj = loadProject(path_integrated_singlesite, str(path_integrated_singlesite_config)) sites = ["M7_4096"] # get decimation, window parameters and selector decParams = getDecimationParameters(4096, proj.config) winParams = getWindowParameters(decParams, proj.config) selector = getWindowSelector( proj, decParams, winParams, proj.config.configParams["Spectra"]["specdir"]) # now add a mask selector.setSites(sites) selector.addWindowMask(sites[0], "coh_tf") selector.calcSharedWindows() declevel = 0 unmaskedWindows = selector.getUnmaskedWindowsLevel(declevel) spectraBatches = selector.getSpecReaderBatches(declevel) for batch in spectraBatches: batch = spectraBatches[0] reader = batch[sites[0]] batchedWindows = unmaskedWindows.intersection( set(range(batch["globalrange"][0], batch["globalrange"][1] + 1))) # read the batch batchData, batchGlobalIndices = reader.readBinaryBatchGlobal( batchedWindows) # for each window, check to make sure all correct for testGlobalIndex in list(batchedWindows): winData1 = reader.readBinaryWindowGlobal(testGlobalIndex) # matching readBatch index matchingBatchIndex = list(batchGlobalIndices).index( testGlobalIndex) winData2 = batchData[matchingBatchIndex] # test winData1 and winData2 chans = winData1.chans for chan in chans: assert np.array_equal(winData1.data[chan], winData2.data[chan])
def test_getTimeData() -> None: """Test getting time data""" from datapaths import path_integrated_singlesite from resistics.project.io import loadProject from resistics.project.io import loadProject from resistics.project.time import getTimeReader from datetime import datetime proj = loadProject(path_integrated_singlesite) reader = getTimeReader(proj, "M7_4096", "meas_2016-02-22_02-00-00") assert reader.getStartDatetime() == datetime(2016, 2, 22, 2, 00, 00, 000000) assert reader.getStopDatetime() == datetime(2016, 2, 22, 2, 29, 59, 999756) assert reader.getNumSamples() == 7372800 assert reader.getSampleFreq() == 4096 assert reader.getChannels() == ["Ex", "Ey", "Hx", "Hy", "Hz"]
dx=60, dy=60.7, ) # headers for telluric only stations from resistics.time.reader_lemib423e import folderB423EHeaders folderB423EHeaders(intersitePath / "timeData" / "site2_te", 500, ex="E1", ey="E2", dx=60, dy=60.7) # load the project proj = loadProject(intersitePath, "customconfig.ini") proj.printInfo() fig = proj.view() fig.savefig(intersiteImages / "timeline.png") # view time from resistics.project.time import viewTime fig = viewTime( proj, startDate="2019-05-27 14:15:00", endDate="2019-05-27 15:00:00", filter={"lpfilt": 4}, save=False, show=False, )
from datapaths import remotePath, remoteImages from resistics.project.io import loadProject from resistics.project.spectra import calculateSpectra from resistics.project.transfunc import processProject, viewImpedance from resistics.project.statistics import ( calculateStatistics, calculateRemoteStatistics, viewStatistic, ) from resistics.project.mask import newMaskData, calculateMask from resistics.common.plot import plotOptionsTransferFunction, getPaperFonts plotOptions = plotOptionsTransferFunction(plotfonts=getPaperFonts()) proj = loadProject(remotePath, "manualWindowsConfig.ini") # calculate spectra calculateSpectra(proj, sites=["M6", "Remote"]) proj.refresh() processProject(proj, sites=["M6", "Remote"]) figs = viewImpedance( proj, sites=["M6", "Remote"], sampleFreqs=[128], oneplot=False, plotoptions=plotOptions, save=False, show=False, ) figs[0].savefig(remoteImages / "singleSiteM6_128_man8_5.png") figs[1].savefig(remoteImages / "singleSiteRemote_128_man8_5.png")
from datapaths import remotePath, remoteImages from resistics.project.io import loadProject from resistics.project.time import preProcess, viewTime proj = loadProject(remotePath) proj.printInfo() # get information about the local site siteLocal = proj.getSiteData("M6") siteLocal.printInfo() # get information about the SPAM data remote reference siteRemote = proj.getSiteData("RemoteSPAM") siteRemote.printInfo() # want to resample the RemoteSPAM data to 128 Hz # SPAM data want to resample onto the second # to make sure the times are matching, we can loop through the 128Hz measurements of M6 and use those times meas128 = siteLocal.getMeasurements(128) for meas in meas128: start = siteLocal.getMeasurementStart(meas).strftime("%Y-%m-%d %H:%M:%S") stop = siteLocal.getMeasurementEnd(meas).strftime("%Y-%m-%d %H:%M:%S") postpend = siteLocal.getMeasurementStart(meas).strftime( "%Y-%m-%d_%H-%M-%S") print("Processing data from {} to {}".format(start, stop)) preProcess( proj, sites="RemoteSPAM", start=start, stop=stop, interp=True, resamp={250: 128}, outputsite="Remote",
from datapaths import projectPath, imagePath from resistics.project.io import loadProject from resistics.project.spectra import calculateSpectra from resistics.project.transfunc import processProject, viewImpedance from resistics.project.statistics import calculateStatistics from resistics.project.mask import newMaskData, calculateMask if __name__ == "__main__": projData = loadProject(projectPath, "multiconfig.ini") # calculate spectrum using standard options calculateSpectra(projData) projData.refresh() calculateStatistics(projData) # process project with standard options processProject(projData) figs = viewImpedance(projData, oneplot=False, show=False, save=False) figs[0].savefig(imagePath / "multproc_standard_process") # calculate mask for 128 maskData = newMaskData(projData, 128) maskData.setStats(["coherence"]) maskData.addConstraint("coherence", { "cohExHy": [0.7, 1.0], "cohEyHx": [0.7, 1.0] }) maskData.maskName = "coh70_100" calculateMask(projData, maskData, sites=["site1"]) # calculate mask for 4096 maskData = newMaskData(projData, 4096)
from datapaths import projectPath, statImagePath from resistics.project.io import loadProject from resistics.project.statistics import getStatisticData from resistics.statistics.utils import getStatNames proj = loadProject(projectPath) stats, remoteStats = getStatNames() statCrossplots = dict() statCrossplots["coherence"] = [["cohExHy", "cohEyHx"], ["cohExHx", "cohEyHy"]] statCrossplots["transferFunction"] = [ ["ExHxReal", "ExHxImag"], ["ExHyReal", "ExHyImag"], ["EyHxReal", "EyHxImag"], ["EyHyReal", "EyHyImag"], ] for stat in stats: statData = getStatisticData(proj, "M1", "meas_2016-03-23_02-00-00", stat) fig = statData.view(0) fig.savefig(statImagePath / "M1_{}_view_4096".format(stat)) fig = statData.histogram(0) fig.savefig(statImagePath / "M1_{}_histogram_4096".format(stat)) if stat in statCrossplots: fig = statData.crossplot(0, crossplots=statCrossplots[stat]) fig.savefig(statImagePath / "M1_{}_crossplot_4096".format(stat)) fig = statData.densityplot(0, crossplots=statCrossplots[stat]) fig.savefig(statImagePath / "M1_{}_densityplot_4096".format(stat)) for stat in stats: statData = getStatisticData(proj, "Remote_M1", "run4_2016-03-25_02-35-00", stat) fig = statData.view(0)
from datapaths import projectPath, imagePath from resistics.project.io import loadProject # load project and configuration file projData = loadProject(projectPath, configFile="tutorialConfig.ini") # get decimation parameters from resistics.project.shortcuts import getDecimationParameters decimationParameters = getDecimationParameters(4096, projData.config) decimationParameters.printInfo() # get the window parameters from resistics.project.shortcuts import getWindowParameters windowParameters = getWindowParameters(decimationParameters, projData.config) windowParameters.printInfo() from resistics.project.shortcuts import getWindowSelector decimationParameters = getDecimationParameters(128, projData.config) windowParameters = getWindowParameters(decimationParameters, projData.config) selector = getWindowSelector(projData, decimationParameters, windowParameters) selector.printInfo() # add a site and print the information to the terminal selector.setSites(["site1"]) selector.printInfo() # calculate shared windows selector.calcSharedWindows()
from datapaths import projectPath, imagePath from resistics.project.io import loadProject projData = loadProject(projectPath) # calculate another set of spectra for the 128 Hz data with notching at 50Hz and 16.667Hz from resistics.project.spectra import calculateSpectra calculateSpectra(projData, sampleFreqs=[128], notch=[50], specdir="notch") projData.refresh() # view the spectra from resistics.common.plot import plotOptionsSpec, getPaperFonts from resistics.project.spectra import viewSpectra, viewSpectraSection plotOptions = plotOptionsSpec(plotfonts=getPaperFonts()) fig = viewSpectra( projData, "site1", "meas_2012-02-10_11-30-00", specdir="notch", plotoptions=plotOptions, show=False, save=False, ) fig.savefig(imagePath / "multspec_viewspec_notch_spec") fig = viewSpectraSection( projData, "site1", "meas_2012-02-10_11-30-00",
from datapaths import projectPath, imagePath from resistics.project.io import loadProject proj = loadProject(projectPath, "customconfig.ini") proj.printInfo() from resistics.project.spectra import calculateSpectra from resistics.project.statistics import calculateStatistics from resistics.project.transfunc import processProject, viewImpedance from resistics.common.plot import plotOptionsTransferFunction, getPaperFonts calculateSpectra(proj) proj.refresh() calculateStatistics(proj) processProject(proj) plotOptions = plotOptionsTransferFunction(figsize=(24, 12), plotfonts=getPaperFonts()) plotOptions["res_ylim"] = [1, 1000000] figs = viewImpedance(proj, oneplot=False, plotoptions=plotOptions, show=False, save=False) figs[0].savefig(imagePath / "impedance_config.png") # now try again with some statistics for the dead band from resistics.project.mask import newMaskData, calculateMask maskData = newMaskData(proj, 500) maskData.setStats(["coherence"]) maskData.addConstraintLevel("coherence", {
from datapaths import remotePath, remoteImages from resistics.project.io import loadProject from resistics.project.spectra import calculateSpectra from resistics.project.transfunc import processProject, viewImpedance from resistics.project.statistics import calculateRemoteStatistics, viewStatistic from resistics.project.mask import newMaskData, calculateMask from resistics.common.plot import plotOptionsTransferFunction, getPaperFonts plotOptions = plotOptionsTransferFunction(plotfonts=getPaperFonts()) proj = loadProject(remotePath, "remoteConfig.ini") calculateSpectra(proj, sites=["M6", "Remote"]) proj.refresh() # single site processing processProject(proj, sites=["M6", "Remote"]) figs = viewImpedance( proj, sites=["M6", "Remote"], sampleFreqs=[128], oneplot=False, plotoptions=plotOptions, save=False, show=False, ) figs[0].savefig(remoteImages / "singleSiteM6_128_dec8_5.png") figs[1].savefig(remoteImages / "singleSiteRemote_128_dec8_5.png") # calculate the statistic we are interested in calculateRemoteStatistics(proj, "Remote", sites=["M6"], sampleFreqs=[128])
from datapaths import preprocessPath, preprocessImages from pathlib import Path from resistics.project.io import loadProject proj = loadProject(preprocessPath) proj.printInfo() from resistics.common.plot import plotOptionsTime, getPresentationFonts plotOptions = plotOptionsTime(plotfonts=getPresentationFonts()) from resistics.time.reader_ats import TimeReaderATS site1 = proj.getSiteData("site1") readerATS = TimeReaderATS( site1.getMeasurementTimePath("meas_2012-02-10_11-05-00")) # headers of recording headers = readerATS.getHeaders() chanHeaders, chanMap = readerATS.getChanHeaders() # separate out two datasets timeOriginal1 = readerATS.getPhysicalData("2012-02-10 11:05:00", "2012-02-10 11:09:00", remaverage=False) timeOriginal2 = readerATS.getPhysicalData("2012-02-10 11:10:00", "2012-02-10 11:14:00", remaverage=False) from resistics.time.writer_internal import TimeWriterInternal # create a new site proj.createSite("site1_gaps")
from datapaths import projectPath, imagePath from resistics.project.io import loadProject # load the project and also provide a config file projData = loadProject(projectPath, configFile="asciiconfig.ini") projData.printInfo() from resistics.project.time import viewTime from resistics.common.plot import plotOptionsTime, getPaperFonts plotOptions = plotOptionsTime(plotfonts=getPaperFonts()) fig = viewTime( projData, "2018-01-03 00:00:00", "2018-01-05 00:00:00", polreverse={"Hy": True}, plotoptions=plotOptions, save=False, ) fig.savefig(imagePath / "viewTime_polreverse") # calculate spectrum using the new configuration from resistics.project.spectra import calculateSpectra calculateSpectra(projData, calibrate=False, polreverse={"Hy": True}) projData.refresh() # plot spectra stack from resistics.project.spectra import viewSpectraStack from resistics.common.plot import plotOptionsSpec, getPaperFonts
def test_masked_processing() -> None: """Test masked processing the project""" from datapaths import path_integrated_singlesite, path_integrated_singlesite_config from resistics.project.io import loadProject from resistics.project.spectra import calculateSpectra from resistics.project.statistics import calculateStatistics from resistics.project.mask import newMaskData, calculateMask from resistics.project.transfunc import processProject from resistics.project.transfunc import getTransferFunctionData, viewImpedance # load project sites = ["M7_4096"] proj = loadProject(path_integrated_singlesite, str(path_integrated_singlesite_config)) # calculateSpectra(proj) # proj.refresh() # calculateStatistics(proj) # calculate out a mask maskData = newMaskData(proj, 4096) maskData.setStats(["coherence", "transferFunction", "resPhase"]) maskData.addConstraintLevel("coherence", { "cohExHy": [0.8, 1.0], "cohEyHx": [0.8, 1.0] }, 0) maskData.addConstraintLevel("coherence", { "cohExHy": [0.8, 1.0], "cohEyHx": [0.8, 1.0] }, 1) maskData.addConstraintLevel("coherence", { "cohExHy": [0.8, 1.0], "cohEyHx": [0.7, 1.0] }, 2) maskData.addConstraintLevel("coherence", { "cohExHy": [0.7, 1.0], "cohEyHx": [0.7, 1.0] }, 3) maskData.addConstraintLevel("coherence", { "cohExHy": [0.7, 1.0], "cohEyHx": [0.7, 1.0] }, 4) maskData.addConstraintFreq("transferFunction", { "EyHxReal": [0, 150], "EyHxImag": [0, 220] }, 0, 4) maskData.addConstraintFreq("transferFunction", { "ExHyReal": [-120, 0], "ExHyImag": [-200, -30] }, 1, 0) maskData.addConstraintFreq("transferFunction", { "ExHyReal": [-100, 0], "ExHyImag": [-160, -30] }, 1, 1) maskData.addConstraintFreq("transferFunction", { "ExHyReal": [-90, 0], "ExHyImag": [-120, -30] }, 1, 2) maskData.addConstraintFreq("transferFunction", { "ExHyReal": [-60, -25], "ExHyImag": [-90, -75] }, 1, 3) maskData.addConstraintFreq("transferFunction", { "ExHyReal": [-80, 0], "ExHyImag": [-100, -20] }, 1, 4) # finally, lets give maskData a name, which will relate to the output file maskData.maskName = "coh_tf" calculateMask(proj, maskData, sites=sites) # process processProject( proj, sites=sites, masks={"M7_4096": "coh_tf"}, postpend="coh_tf", ) # tf = getTransferFunctionData(proj, "M7_4096", 4096) # test the transfer function viewImpedance( proj, sites=sites, postpend="coh_tf", polarisations=["ExHy", "EyHx"], oneplot=False, show=False, save=True, )