def test_local_regression() -> None:
"""Test selecting masked windows"""
from datapaths import path_integrated_singlesite, path_integrated_singlesite_config
from resistics.project.io import loadProject
from resistics.project.transfunc import getTransferFunctionData, viewImpedance
from resistics.project.shortcuts import (
getWindowSelector,
getDecimationParameters,
getWindowParameters,
)
from localtest2 import LocalRegressor
# from localtest import LocalRegressor
import numpy as np
proj = loadProject(path_integrated_singlesite,
str(path_integrated_singlesite_config))
sites = ["M7_4096"]
decParams = getDecimationParameters(4096, proj.config)
winParams = getWindowParameters(decParams, proj.config)
selector = getWindowSelector(
proj, decParams, winParams,
proj.config.configParams["Spectra"]["specdir"])
# now add a mask
selector.setSites(sites + sites)
selector.addWindowMask(sites[0], "coh_tf")
selector.calcSharedWindows()
# add the input and output site
processor = LocalRegressor(selector, "")
processor.setInput(sites[0], ["Hx", "Hy"])
processor.setOutput(sites[0], ["Ex", "Ey"])
processor.postpend = "stacked"
processor.printInfo()
processor.process(4)
def test_statistic_transfunc() -> None:
"""Get the transfer functions statistic"""
from datapaths import path_integrated_singlesite, path_integrated_singlesite_config
from resistics.project.io import loadProject
from resistics.project.statistics import getStatisticData
# load project
sites = ["M7_4096"]
proj = loadProject(path_integrated_singlesite,
str(path_integrated_singlesite_config))
statData = getStatisticData(proj, sites[0], "meas_2016-02-25_02-00-00",
"transferFunction", 1)
data = statData.getStatLocal(0)
print(statData.winStats)
print(data)
# now do the same window through the
from resistics.project.transfunc import getTransferFunctionData
from resistics.project.shortcuts import (
getWindowSelector,
getDecimationParameters,
getWindowParameters,
)
from localtest3 import LocalRegressor
import numpy as np
proj = loadProject(path_integrated_singlesite,
str(path_integrated_singlesite_config))
sites = ["M7_4096"]
decParams = getDecimationParameters(4096, proj.config)
winParams = getWindowParameters(decParams, proj.config)
selector = getWindowSelector(
proj, decParams, winParams,
proj.config.configParams["Spectra"]["specdir"])
# now add a mask
selector.setSites(sites + sites)
selector.calcSharedWindows()
# add the input and output site
processor = LocalRegressor(selector, "")
processor.setInput(sites[0], ["Hx", "Hy"])
processor.setOutput(sites[0], ["Ex", "Ey"])
processor.postpend = "check"
processor.process(4)
def test_loadProject() -> None:
"""Test loading the project"""
from datapaths import path_integrated_singlesite
from resistics.project.io import loadProject
proj = loadProject(path_integrated_singlesite)
assert proj.refTime.strftime("%Y-%m-%d %H:%M:%S.%f") == "2016-02-18 12:00:00.000000"
assert (
proj.projStart.strftime("%Y-%m-%d %H:%M:%S.%f") == "2016-02-19 11:22:57.000000"
)
assert proj.projEnd.strftime("%Y-%m-%d %H:%M:%S.%f") == "2016-02-25 02:29:59.999756"
assert proj.sites == ["M7_4096"]
def test_window_selector() -> None:
"""Test selecting masked windows"""
from datapaths import path_integrated_singlesite, path_integrated_singlesite_config
from resistics.project.io import loadProject
from resistics.project.shortcuts import (
getWindowSelector,
getDecimationParameters,
getWindowParameters,
)
import numpy as np
proj = loadProject(path_integrated_singlesite,
str(path_integrated_singlesite_config))
sites = ["M7_4096"]
# get decimation, window parameters and selector
decParams = getDecimationParameters(4096, proj.config)
winParams = getWindowParameters(decParams, proj.config)
selector = getWindowSelector(
proj, decParams, winParams,
proj.config.configParams["Spectra"]["specdir"])
# now add a mask
selector.setSites(sites)
selector.addWindowMask(sites[0], "coh_tf")
selector.calcSharedWindows()
declevel = 0
unmaskedWindows = selector.getUnmaskedWindowsLevel(declevel)
spectraBatches = selector.getSpecReaderBatches(declevel)
for batch in spectraBatches:
batch = spectraBatches[0]
reader = batch[sites[0]]
batchedWindows = unmaskedWindows.intersection(
set(range(batch["globalrange"][0], batch["globalrange"][1] + 1)))
# read the batch
batchData, batchGlobalIndices = reader.readBinaryBatchGlobal(
batchedWindows)
# for each window, check to make sure all correct
for testGlobalIndex in list(batchedWindows):
winData1 = reader.readBinaryWindowGlobal(testGlobalIndex)
# matching readBatch index
matchingBatchIndex = list(batchGlobalIndices).index(
testGlobalIndex)
winData2 = batchData[matchingBatchIndex]
# test winData1 and winData2
chans = winData1.chans
for chan in chans:
assert np.array_equal(winData1.data[chan], winData2.data[chan])
def test_getTimeData() -> None:
"""Test getting time data"""
from datapaths import path_integrated_singlesite
from resistics.project.io import loadProject
from resistics.project.io import loadProject
from resistics.project.time import getTimeReader
from datetime import datetime
proj = loadProject(path_integrated_singlesite)
reader = getTimeReader(proj, "M7_4096", "meas_2016-02-22_02-00-00")
assert reader.getStartDatetime() == datetime(2016, 2, 22, 2, 00, 00,
000000)
assert reader.getStopDatetime() == datetime(2016, 2, 22, 2, 29, 59, 999756)
assert reader.getNumSamples() == 7372800
assert reader.getSampleFreq() == 4096
assert reader.getChannels() == ["Ex", "Ey", "Hx", "Hy", "Hz"]
dx=60,
dy=60.7,
)
# headers for telluric only stations
from resistics.time.reader_lemib423e import folderB423EHeaders
folderB423EHeaders(intersitePath / "timeData" / "site2_te",
500,
ex="E1",
ey="E2",
dx=60,
dy=60.7)
# load the project
proj = loadProject(intersitePath, "customconfig.ini")
proj.printInfo()
fig = proj.view()
fig.savefig(intersiteImages / "timeline.png")
# view time
from resistics.project.time import viewTime
fig = viewTime(
proj,
startDate="2019-05-27 14:15:00",
endDate="2019-05-27 15:00:00",
filter={"lpfilt": 4},
save=False,
show=False,
)
from datapaths import remotePath, remoteImages
from resistics.project.io import loadProject
from resistics.project.spectra import calculateSpectra
from resistics.project.transfunc import processProject, viewImpedance
from resistics.project.statistics import (
calculateStatistics,
calculateRemoteStatistics,
viewStatistic,
)
from resistics.project.mask import newMaskData, calculateMask
from resistics.common.plot import plotOptionsTransferFunction, getPaperFonts
plotOptions = plotOptionsTransferFunction(plotfonts=getPaperFonts())
proj = loadProject(remotePath, "manualWindowsConfig.ini")
# calculate spectra
calculateSpectra(proj, sites=["M6", "Remote"])
proj.refresh()
processProject(proj, sites=["M6", "Remote"])
figs = viewImpedance(
proj,
sites=["M6", "Remote"],
sampleFreqs=[128],
oneplot=False,
plotoptions=plotOptions,
save=False,
show=False,
)
figs[0].savefig(remoteImages / "singleSiteM6_128_man8_5.png")
figs[1].savefig(remoteImages / "singleSiteRemote_128_man8_5.png")
from datapaths import remotePath, remoteImages
from resistics.project.io import loadProject
from resistics.project.time import preProcess, viewTime
proj = loadProject(remotePath)
proj.printInfo()
# get information about the local site
siteLocal = proj.getSiteData("M6")
siteLocal.printInfo()
# get information about the SPAM data remote reference
siteRemote = proj.getSiteData("RemoteSPAM")
siteRemote.printInfo()
# want to resample the RemoteSPAM data to 128 Hz
# SPAM data want to resample onto the second
# to make sure the times are matching, we can loop through the 128Hz measurements of M6 and use those times
meas128 = siteLocal.getMeasurements(128)
for meas in meas128:
start = siteLocal.getMeasurementStart(meas).strftime("%Y-%m-%d %H:%M:%S")
stop = siteLocal.getMeasurementEnd(meas).strftime("%Y-%m-%d %H:%M:%S")
postpend = siteLocal.getMeasurementStart(meas).strftime(
"%Y-%m-%d_%H-%M-%S")
print("Processing data from {} to {}".format(start, stop))
preProcess(
proj,
sites="RemoteSPAM",
start=start,
stop=stop,
interp=True,
resamp={250: 128},
outputsite="Remote",
from datapaths import projectPath, imagePath
from resistics.project.io import loadProject
from resistics.project.spectra import calculateSpectra
from resistics.project.transfunc import processProject, viewImpedance
from resistics.project.statistics import calculateStatistics
from resistics.project.mask import newMaskData, calculateMask
if __name__ == "__main__":
projData = loadProject(projectPath, "multiconfig.ini")
# calculate spectrum using standard options
calculateSpectra(projData)
projData.refresh()
calculateStatistics(projData)
# process project with standard options
processProject(projData)
figs = viewImpedance(projData, oneplot=False, show=False, save=False)
figs[0].savefig(imagePath / "multproc_standard_process")
# calculate mask for 128
maskData = newMaskData(projData, 128)
maskData.setStats(["coherence"])
maskData.addConstraint("coherence", {
"cohExHy": [0.7, 1.0],
"cohEyHx": [0.7, 1.0]
})
maskData.maskName = "coh70_100"
calculateMask(projData, maskData, sites=["site1"])
# calculate mask for 4096
maskData = newMaskData(projData, 4096)
from datapaths import projectPath, statImagePath
from resistics.project.io import loadProject
from resistics.project.statistics import getStatisticData
from resistics.statistics.utils import getStatNames
proj = loadProject(projectPath)
stats, remoteStats = getStatNames()
statCrossplots = dict()
statCrossplots["coherence"] = [["cohExHy", "cohEyHx"], ["cohExHx", "cohEyHy"]]
statCrossplots["transferFunction"] = [
["ExHxReal", "ExHxImag"],
["ExHyReal", "ExHyImag"],
["EyHxReal", "EyHxImag"],
["EyHyReal", "EyHyImag"],
]
for stat in stats:
statData = getStatisticData(proj, "M1", "meas_2016-03-23_02-00-00", stat)
fig = statData.view(0)
fig.savefig(statImagePath / "M1_{}_view_4096".format(stat))
fig = statData.histogram(0)
fig.savefig(statImagePath / "M1_{}_histogram_4096".format(stat))
if stat in statCrossplots:
fig = statData.crossplot(0, crossplots=statCrossplots[stat])
fig.savefig(statImagePath / "M1_{}_crossplot_4096".format(stat))
fig = statData.densityplot(0, crossplots=statCrossplots[stat])
fig.savefig(statImagePath / "M1_{}_densityplot_4096".format(stat))
for stat in stats:
statData = getStatisticData(proj, "Remote_M1", "run4_2016-03-25_02-35-00", stat)
fig = statData.view(0)
from datapaths import projectPath, imagePath from resistics.project.io import loadProject # load project and configuration file projData = loadProject(projectPath, configFile="tutorialConfig.ini") # get decimation parameters from resistics.project.shortcuts import getDecimationParameters decimationParameters = getDecimationParameters(4096, projData.config) decimationParameters.printInfo() # get the window parameters from resistics.project.shortcuts import getWindowParameters windowParameters = getWindowParameters(decimationParameters, projData.config) windowParameters.printInfo() from resistics.project.shortcuts import getWindowSelector decimationParameters = getDecimationParameters(128, projData.config) windowParameters = getWindowParameters(decimationParameters, projData.config) selector = getWindowSelector(projData, decimationParameters, windowParameters) selector.printInfo() # add a site and print the information to the terminal selector.setSites(["site1"]) selector.printInfo() # calculate shared windows selector.calcSharedWindows()
from datapaths import projectPath, imagePath
from resistics.project.io import loadProject
projData = loadProject(projectPath)
# calculate another set of spectra for the 128 Hz data with notching at 50Hz and 16.667Hz
from resistics.project.spectra import calculateSpectra
calculateSpectra(projData, sampleFreqs=[128], notch=[50], specdir="notch")
projData.refresh()
# view the spectra
from resistics.common.plot import plotOptionsSpec, getPaperFonts
from resistics.project.spectra import viewSpectra, viewSpectraSection
plotOptions = plotOptionsSpec(plotfonts=getPaperFonts())
fig = viewSpectra(
projData,
"site1",
"meas_2012-02-10_11-30-00",
specdir="notch",
plotoptions=plotOptions,
show=False,
save=False,
)
fig.savefig(imagePath / "multspec_viewspec_notch_spec")
fig = viewSpectraSection(
projData,
"site1",
"meas_2012-02-10_11-30-00",
from datapaths import projectPath, imagePath
from resistics.project.io import loadProject
proj = loadProject(projectPath, "customconfig.ini")
proj.printInfo()
from resistics.project.spectra import calculateSpectra
from resistics.project.statistics import calculateStatistics
from resistics.project.transfunc import processProject, viewImpedance
from resistics.common.plot import plotOptionsTransferFunction, getPaperFonts
calculateSpectra(proj)
proj.refresh()
calculateStatistics(proj)
processProject(proj)
plotOptions = plotOptionsTransferFunction(figsize=(24, 12),
plotfonts=getPaperFonts())
plotOptions["res_ylim"] = [1, 1000000]
figs = viewImpedance(proj,
oneplot=False,
plotoptions=plotOptions,
show=False,
save=False)
figs[0].savefig(imagePath / "impedance_config.png")
# now try again with some statistics for the dead band
from resistics.project.mask import newMaskData, calculateMask
maskData = newMaskData(proj, 500)
maskData.setStats(["coherence"])
maskData.addConstraintLevel("coherence", {
from datapaths import remotePath, remoteImages
from resistics.project.io import loadProject
from resistics.project.spectra import calculateSpectra
from resistics.project.transfunc import processProject, viewImpedance
from resistics.project.statistics import calculateRemoteStatistics, viewStatistic
from resistics.project.mask import newMaskData, calculateMask
from resistics.common.plot import plotOptionsTransferFunction, getPaperFonts
plotOptions = plotOptionsTransferFunction(plotfonts=getPaperFonts())
proj = loadProject(remotePath, "remoteConfig.ini")
calculateSpectra(proj, sites=["M6", "Remote"])
proj.refresh()
# single site processing
processProject(proj, sites=["M6", "Remote"])
figs = viewImpedance(
proj,
sites=["M6", "Remote"],
sampleFreqs=[128],
oneplot=False,
plotoptions=plotOptions,
save=False,
show=False,
)
figs[0].savefig(remoteImages / "singleSiteM6_128_dec8_5.png")
figs[1].savefig(remoteImages / "singleSiteRemote_128_dec8_5.png")
# calculate the statistic we are interested in
calculateRemoteStatistics(proj, "Remote", sites=["M6"], sampleFreqs=[128])
from datapaths import preprocessPath, preprocessImages
from pathlib import Path
from resistics.project.io import loadProject
proj = loadProject(preprocessPath)
proj.printInfo()
from resistics.common.plot import plotOptionsTime, getPresentationFonts
plotOptions = plotOptionsTime(plotfonts=getPresentationFonts())
from resistics.time.reader_ats import TimeReaderATS
site1 = proj.getSiteData("site1")
readerATS = TimeReaderATS(
site1.getMeasurementTimePath("meas_2012-02-10_11-05-00"))
# headers of recording
headers = readerATS.getHeaders()
chanHeaders, chanMap = readerATS.getChanHeaders()
# separate out two datasets
timeOriginal1 = readerATS.getPhysicalData("2012-02-10 11:05:00",
"2012-02-10 11:09:00",
remaverage=False)
timeOriginal2 = readerATS.getPhysicalData("2012-02-10 11:10:00",
"2012-02-10 11:14:00",
remaverage=False)
from resistics.time.writer_internal import TimeWriterInternal
# create a new site
proj.createSite("site1_gaps")
from datapaths import projectPath, imagePath
from resistics.project.io import loadProject
# load the project and also provide a config file
projData = loadProject(projectPath, configFile="asciiconfig.ini")
projData.printInfo()
from resistics.project.time import viewTime
from resistics.common.plot import plotOptionsTime, getPaperFonts
plotOptions = plotOptionsTime(plotfonts=getPaperFonts())
fig = viewTime(
projData,
"2018-01-03 00:00:00",
"2018-01-05 00:00:00",
polreverse={"Hy": True},
plotoptions=plotOptions,
save=False,
)
fig.savefig(imagePath / "viewTime_polreverse")
# calculate spectrum using the new configuration
from resistics.project.spectra import calculateSpectra
calculateSpectra(projData, calibrate=False, polreverse={"Hy": True})
projData.refresh()
# plot spectra stack
from resistics.project.spectra import viewSpectraStack
from resistics.common.plot import plotOptionsSpec, getPaperFonts
def test_masked_processing() -> None:
"""Test masked processing the project"""
from datapaths import path_integrated_singlesite, path_integrated_singlesite_config
from resistics.project.io import loadProject
from resistics.project.spectra import calculateSpectra
from resistics.project.statistics import calculateStatistics
from resistics.project.mask import newMaskData, calculateMask
from resistics.project.transfunc import processProject
from resistics.project.transfunc import getTransferFunctionData, viewImpedance
# load project
sites = ["M7_4096"]
proj = loadProject(path_integrated_singlesite,
str(path_integrated_singlesite_config))
# calculateSpectra(proj)
# proj.refresh()
# calculateStatistics(proj)
# calculate out a mask
maskData = newMaskData(proj, 4096)
maskData.setStats(["coherence", "transferFunction", "resPhase"])
maskData.addConstraintLevel("coherence", {
"cohExHy": [0.8, 1.0],
"cohEyHx": [0.8, 1.0]
}, 0)
maskData.addConstraintLevel("coherence", {
"cohExHy": [0.8, 1.0],
"cohEyHx": [0.8, 1.0]
}, 1)
maskData.addConstraintLevel("coherence", {
"cohExHy": [0.8, 1.0],
"cohEyHx": [0.7, 1.0]
}, 2)
maskData.addConstraintLevel("coherence", {
"cohExHy": [0.7, 1.0],
"cohEyHx": [0.7, 1.0]
}, 3)
maskData.addConstraintLevel("coherence", {
"cohExHy": [0.7, 1.0],
"cohEyHx": [0.7, 1.0]
}, 4)
maskData.addConstraintFreq("transferFunction", {
"EyHxReal": [0, 150],
"EyHxImag": [0, 220]
}, 0, 4)
maskData.addConstraintFreq("transferFunction", {
"ExHyReal": [-120, 0],
"ExHyImag": [-200, -30]
}, 1, 0)
maskData.addConstraintFreq("transferFunction", {
"ExHyReal": [-100, 0],
"ExHyImag": [-160, -30]
}, 1, 1)
maskData.addConstraintFreq("transferFunction", {
"ExHyReal": [-90, 0],
"ExHyImag": [-120, -30]
}, 1, 2)
maskData.addConstraintFreq("transferFunction", {
"ExHyReal": [-60, -25],
"ExHyImag": [-90, -75]
}, 1, 3)
maskData.addConstraintFreq("transferFunction", {
"ExHyReal": [-80, 0],
"ExHyImag": [-100, -20]
}, 1, 4)
# finally, lets give maskData a name, which will relate to the output file
maskData.maskName = "coh_tf"
calculateMask(proj, maskData, sites=sites)
# process
processProject(
proj,
sites=sites,
masks={"M7_4096": "coh_tf"},
postpend="coh_tf",
)
# tf = getTransferFunctionData(proj, "M7_4096", 4096)
# test the transfer function
viewImpedance(
proj,
sites=sites,
postpend="coh_tf",
polarisations=["ExHy", "EyHx"],
oneplot=False,
show=False,
save=True,
)
