def printList(self) -> List[str]:
"""Class information as a list of strings
Returns
-------
out : List[str]
List of strings with information
"""
textLst: List[str] = []
textLst.append("Time data path = {}".format(self.timePath))
textLst.append("Spectra data path = {}".format(self.specPath))
textLst.append("Statistics data path = {}".format(self.statPath))
textLst.append("Mask data path = {}".format(self.maskPath))
textLst.append("TransFunc data path = {}".format(self.transFuncPath))
textLst.append("Calibration data path = {}".format(self.calPath))
textLst.append("Images data path = {}".format(self.imagePath))
textLst.append("Reference time = {}".format(
self.refTime.strftime("%Y-%m-%d %H:%M:%S")))
textLst.append("Project start time = {}".format(
self.projStart.strftime("%Y-%m-%d %H:%M:%S.%f")))
textLst.append("Project stop time = {}".format(
self.projEnd.strftime("%Y-%m-%d %H:%M:%S.%f")))
textLst.append("Project found {} sites:".format(self.getNumSites()))
for site in self.sites:
textLst.append("{}\t\tstart: {}\tend: {}".format(
site,
self.getSiteData(site).siteStart,
self.getSiteData(site).siteEnd,
))
textLst.append("Sampling frequencies found in project (Hz): {}".format(
listToString(self.getSampleFreqs())))
return textLst
def write(self):
"""Write transfer function data in internal format file"""
outF = open(self.filepath, "w")
self.printText("Writing out transfer function data to file {}".format(
self.filepath))
# write out the headers
for h, v in self.headers.items():
if isinstance(v, list):
v = listToString(v)
if len(v) == 0:
v = "False"
outF.write("{} = {}\n".format(h, v))
# write evaluation frequencies
outF.write("Evaluation frequencies\n")
outF.write("{}\n".format(arrayToString(self.tfData.freq)))
# now need to write out the other data
for pol in self.polarisations:
outF.write("Z-{}\n".format(pol))
outF.write("{}\n".format(arrayToString(self.tfData.data[pol])))
# variances
for pol in self.polarisations:
outF.write("Var-{}\n".format(pol))
outF.write("{}\n".format(arrayToString(
self.tfData.variances[pol])))
outF.close()
def printList(self) -> List[str]:
"""Class information as a list of strings
Returns
-------
out : list
List of strings with information
"""
textLst = []
textLst.append("Default options")
textLst.append("\tInput Chans = {}".format(listToString(self.getInChans())))
textLst.append("\tOutput Chans = {}".format(listToString(self.getOutChans())))
textLst.append(
"\tRemote Chans = {}".format(listToString(self.getRemoteChans()))
)
textLst.append("\tPowers = {}".format(listToString(self.getPSDChans())))
textLst.append(
"\tCoherence pairs = {}".format(listToString(self.getCohPairs()))
)
textLst.append(
"\tPartial coherence = {}".format(listToString(self.getPolDirs()))
)
if len(self.getEvalFreq()) == 0:
textLst.append("Evaluation frequencies = {}")
else:
textLst.append(
"Evaluation frequencies = {}".format(arrayToString(self.getEvalFreq()))
)
return textLst
def write(self, maskData: MaskData) -> None:
"""Write the maskData out to datapath
Mask data is saved as a numpy binary object
Parameters
----------
maskData : MaskData
MaskData object
"""
infoName, winName = self.getFileNames(maskData.maskName,
maskData.sampleFreq)
infoFile = open(infoName, "w")
# first write out constraints
infoFile.write("{:.9f}\n".format(maskData.sampleFreq))
infoFile.write("{}\n".format(maskData.numLevels))
for iL in range(0, maskData.numLevels):
infoFile.write("{}\n".format(listToString(maskData.evalFreq[iL])))
infoFile.write("{}\n".format(", ".join(maskData.stats)))
# now write out the data file
# first get a sorted list of all the evaluations frequencies to loop through
evalFreq = sorted(list(maskData.constraints.keys()))
for eFreq in evalFreq:
infoFile.write("Frequency = {:.9f}\n".format(eFreq))
for stat in maskData.stats:
infoFile.write("Statistic = {}\n".format(stat))
for component in maskData.constraints[eFreq][stat]:
minVal = maskData.constraints[eFreq][stat][component][0]
maxVal = maskData.constraints[eFreq][stat][component][1]
infoFile.write("{}\t{}\t{}\t{}\n".format(
component,
minVal,
maxVal,
maskData.insideOut[eFreq][stat][component],
))
# then loop through each
infoFile.close()
maskSize = 0
for eFreq in evalFreq:
if len(maskData.maskWindows[eFreq]) > maskSize:
maskSize = len(maskData.maskWindows[eFreq])
# create window mask array and initalise to -1
winMaskArray = np.ones(shape=(len(evalFreq), maskSize), dtype=int) * -1
# now fill the array
for eIdx, eFreq in enumerate(evalFreq):
lst = list(maskData.maskWindows[eFreq])
winMaskArray[eIdx, 0:len(lst)] = lst
np.save(winName, winMaskArray)
def printListSection(self, section: str) -> List[str]:
"""Configuration section information as a list of strings
Returns
-------
out : List[str]
List of strings with information
"""
textLst: List[str] = []
textLst.append("{:s}:".format(section))
for key, value in self.configParams[section].items():
textLst.append("\t{:s} = {}".format(key, value))
defaultOptions = "No defaults used"
if len(self.configParams[section].defaults) > 0:
defaultOptions = listToString(self.configParams[section].defaults)
textLst.append("\tDefaulted options = {:s}".format(defaultOptions))
return textLst
def viewImpedance(self, **kwargs) -> plt.figure:
"""Plots the transfer function data
For resistivity data, both axes are log scale (period and resistivity). For phase data, period is in log scale and phase is linear scale.
Units, x axis is seconds, resistivity is Ohm m and phase is degrees.
Parameters
----------
polarisations : List[str], optional
Polarisations to plot
fig : matplotlib.pyplot.figure, optional
A figure object
oneplot : bool, optional
Boolean flag for plotting all polarisations on one plot rather than separate plots
colours : Dict[str, str], optional
Colours dictionary for plotting impedance components
mk : str, optional
Plot marker type
ls : str, optional
Line style
plotfonts : Dict, optional
A dictionary of plot fonts
label : str, optional
Label for the plots
xlim : List, optional
Limits for the x axis
res_ylim : List, optional
Limits for the resistivity y axis
phase_ylim : List, optional
Limits for the phase y axis
Returns
-------
plt.figure
Matplotlib figure object
"""
polarisations = (kwargs["polarisations"]
if "polarisations" in kwargs else self.polarisations)
# limits
xlim = kwargs["xlim"] if "xlim" in kwargs else [1e-3, 1e4]
res_ylim = kwargs["res_ylim"] if "res_ylim" in kwargs else [1e-2, 1e3]
phase_ylim = kwargs["phase_ylim"] if "phase_ylim" in kwargs else [
-30, 120
]
# markers
colours = (kwargs["colours"]
if "colours" in kwargs else transferFunctionColours())
mk = kwargs["mk"] if "mk" in kwargs else "o"
ls = kwargs["ls"] if "ls" in kwargs else "none"
plotfonts = kwargs[
"plotfonts"] if "plotfonts" in kwargs else getViewFonts()
# calculate number of rows and columns
oneplot = False
if "oneplot" in kwargs and kwargs["oneplot"]:
oneplot = True
nrows = 2
ncols = 1 if oneplot else len(polarisations)
# a multiplier to make sure all the components end up on the right plot
plotNumMult = 0 if ncols > 1 else 1
# plot
if "fig" in kwargs:
fig = plt.figure(kwargs["fig"].number)
else:
figsize = getTransferFunctionFigSize(oneplot, len(polarisations))
fig = plt.figure(figsize=figsize)
st = fig.suptitle(
"Impedance tensor apparent resistivity and phase",
fontsize=plotfonts["suptitle"],
)
st.set_y(0.98)
for idx, pol in enumerate(polarisations):
res, phase = self.getResAndPhase(pol)
resError, phaseError = self.getResAndPhaseErrors(pol)
label = kwargs["label"] + " - {}".format(
pol) if "label" in kwargs else pol
# plot resistivity
ax1 = plt.subplot(nrows, ncols, idx + 1 - plotNumMult * idx)
# the title
if not oneplot:
plt.title("Polarisation {}".format(pol),
fontsize=plotfonts["title"])
else:
plt.title(
"Polarisations {}".format(listToString(polarisations)),
fontsize=plotfonts["title"],
)
# plot the data
ax1.errorbar(
self.period,
res,
yerr=resError,
ls=ls,
marker=mk,
markersize=9,
markerfacecolor="white",
markeredgecolor=colours[pol],
mew=1.1,
color=colours[pol],
ecolor=colours[pol],
elinewidth=1.0,
capsize=4,
barsabove=False,
label=label,
)
ax1.set_xscale("log")
ax1.set_yscale("log")
ax1.set_aspect("equal", adjustable="box")
# axis options
plt.ylabel("Apparent Res. [Ohm m]",
fontsize=plotfonts["axisLabel"])
plt.xlim(xlim)
plt.ylim(res_ylim)
# set tick sizes
for lab in ax1.get_xticklabels() + ax1.get_yticklabels():
lab.set_fontsize(plotfonts["axisTicks"])
# plot phase
ax2 = plt.subplot(nrows, ncols,
ncols + idx + 1 - plotNumMult * idx)
# plot the data
ax2.errorbar(
self.period,
phase,
yerr=phaseError,
ls="none",
marker=mk,
markersize=9,
markerfacecolor="white",
markeredgecolor=colours[pol],
mew=1.1,
color=colours[pol],
ecolor=colours[pol],
elinewidth=1.0,
capsize=4,
barsabove=False,
label=label,
)
ax2.set_xscale("log")
# axis options
plt.xlabel("Period [s]", fontsize=plotfonts["axisLabel"])
plt.ylabel("Phase [degrees]", fontsize=plotfonts["axisLabel"])
plt.xlim(xlim)
plt.ylim(phase_ylim)
# set tick sizes
for lab in ax2.get_xticklabels() + ax2.get_yticklabels():
lab.set_fontsize(plotfonts["axisTicks"])
# add the legend
for idx, pol in enumerate(polarisations):
ax1 = plt.subplot(nrows, ncols, idx + 1 - plotNumMult * idx)
leg = plt.legend(loc="lower left", fontsize=plotfonts["legend"])
leg.get_frame().set_linewidth(0.0)
leg.get_frame().set_facecolor("w")
plt.grid(True, ls="--")
ax2 = plt.subplot(nrows, ncols,
ncols + idx + 1 - plotNumMult * idx)
leg = plt.legend(loc="lower left", fontsize=plotfonts["legend"])
leg.get_frame().set_linewidth(0.0)
leg.get_frame().set_facecolor("w")
plt.grid(True, ls="--")
# show if the figure is not in keywords
if "fig" not in kwargs:
# layout options
plt.tight_layout()
fig.subplots_adjust(top=0.92)
plt.show()
return fig
def calculateRemoteStatistics(projData: ProjectData, remoteSite: str,
**kwargs):
"""Calculate statistics involving a remote reference site
Parameters
----------
projData : ProjectData
A project data instance
remoteSite : str
The name of the site to use as the remote site
sites : List[str], optional
A list of sites to calculate statistics for
sampleFreqs : List[float], optional
List of sampling frequencies for which to calculate statistics
chans : List[str], optional
List of data channels to use
specdir : str, optional
The spectra directory for which to calculate statistics
remotestats : List[str], optional
The statistics to calculate out. Acceptable statistics are: "RR_coherence", "RR_coherenceEqn", "RR_absvalEqn", "RR_transferFunction", "RR_resPhase". Configuration file values are used by default.
"""
options = {}
options["sites"] = projData.getSites()
options["sampleFreqs"] = projData.getSampleFreqs()
options["chans"] = []
options["specdir"] = projData.config.configParams["Spectra"]["specdir"]
options["remotestats"] = projData.config.configParams["Statistics"][
"remotestats"]
options = parseKeywords(options, kwargs)
projectText(
"Calculating stats: {} for sites: {} with remote site {}".format(
listToString(options["remotestats"]),
listToString(options["sites"]),
remoteSite,
))
# create the statistic calculator and IO object
statCalculator = StatisticCalculator()
statIO = StatisticIO()
# loop over sites
for site in options["sites"]:
siteData = projData.getSiteData(site)
measurements = siteData.getMeasurements()
for meas in measurements:
sampleFreq = siteData.getMeasurementSampleFreq(meas)
if sampleFreq not in options["sampleFreqs"]:
# don't need to calculate statistics for this sampling frequency
continue
projectText(
"Calculating stats for site {}, measurement {} with reference {}"
.format(site, meas, remoteSite))
# decimation and window parameters
decParams = getDecimationParameters(sampleFreq, projData.config)
decParams.printInfo()
numLevels = decParams.numLevels
winParams = getWindowParameters(decParams, projData.config)
# create the window selector and find the shared windows
winSelector = getWindowSelector(projData, decParams, winParams)
winSelector.setSites([site, remoteSite])
# calc shared windows between site and remote
winSelector.calcSharedWindows()
# create the spectrum reader
specReader = SpectrumReader(
os.path.join(siteData.getMeasurementSpecPath(meas),
options["specdir"]))
# loop through decimation levels
for iDec in range(0, numLevels):
# open the spectra file for the current decimation level
check = specReader.openBinaryForReading("spectra", iDec)
if not check:
# probably because this decimation level not calculated
continue
specReader.printInfo()
# get a set of the shared windows at this decimation level
# these are the global indices
sharedWindows = winSelector.getSharedWindowsLevel(iDec)
# get other information regarding only this spectra file
refTime = specReader.getReferenceTime()
winSize = specReader.getWindowSize()
winOlap = specReader.getWindowOverlap()
numWindows = specReader.getNumWindows()
evalFreq = decParams.getEvalFrequenciesForLevel(iDec)
sampleFreqDec = specReader.getSampleFreq()
globalOffset = specReader.getGlobalOffset()
fArray = specReader.getFrequencyArray()
# now want to find the size of the intersection between the windows in this file and the shared windows
sharedWindowsMeas = sharedWindows.intersection(
set(np.arange(globalOffset, globalOffset + numWindows)))
sharedWindowsMeas = sorted(list(sharedWindowsMeas))
numSharedWindows = len(sharedWindowsMeas)
statHandlers = {}
# create the statistic handlers
for stat in options["remotestats"]:
statElements = getStatElements(stat)
statHandlers[stat] = StatisticData(stat, refTime,
sampleFreqDec, winSize,
winOlap)
# remember, this is with the remote reference, so the number of windows is number of shared windows
statHandlers[stat].setStatParams(numSharedWindows,
statElements, evalFreq)
statHandlers[stat].comments = specReader.getComments()
statHandlers[stat].addComment(
projData.config.getConfigComment())
statHandlers[stat].addComment(
"Calculating remote statistic: {}".format(stat))
statHandlers[stat].addComment(
"Statistic components: {}".format(
listToString(statElements)))
# loop over the shared windows between the remote station and local station
for iW, globalWindow in enumerate(sharedWindowsMeas):
# get data and set in the statCalculator
winData = specReader.readBinaryWindowGlobal(globalWindow)
statCalculator.setSpectra(fArray, winData, evalFreq)
# for the remote site, use the reader in win selector
remoteSF, remoteReader = winSelector.getSpecReaderForWindow(
remoteSite, iDec, globalWindow)
winDataRR = remoteReader.readBinaryWindowGlobal(
globalWindow)
statCalculator.addRemoteSpec(winDataRR)
for sH in statHandlers:
data = statCalculator.getDataForStatName(sH)
statHandlers[sH].addStat(iW, globalWindow, data)
# save statistic
for sH in statHandlers:
statIO.setDatapath(
os.path.join(siteData.getMeasurementStatPath(meas),
options["specdir"]))
statIO.write(statHandlers[sH], iDec)
def calculateStatistics(projData: ProjectData, **kwargs):
"""Calculate statistics for sites
Parameters
----------
projData : ProjectData
A project data instance
sites : List[str], optional
A list of sites to calculate statistics for
sampleFreqs : List[float], optional
List of sampling frequencies for which to calculate statistics
chans : List[str], optional
List of data channels to use
specdir : str, optional
The spectra directory for which to calculate statistics
stats : List[str], optional
The statistics to calculate out. Acceptable values are: "absvalEqn" "coherence", "psd", "poldir", "transFunc", "resPhase", "partialcoh". Configuration file values are used by default.
"""
options = {}
options["sites"] = projData.getSites()
options["sampleFreqs"] = projData.getSampleFreqs()
options["chans"] = []
options["specdir"] = projData.config.configParams["Spectra"]["specdir"]
options["stats"] = projData.config.configParams["Statistics"]["stats"]
options = parseKeywords(options, kwargs)
projectText("Calculating stats: {} for sites: {}".format(
listToString(options["stats"]), listToString(options["sites"])))
# create the statistic calculator and IO object
statCalculator = StatisticCalculator()
statIO = StatisticIO()
# loop through sites and calculate statistics
for site in options["sites"]:
siteData = projData.getSiteData(site)
measurements = siteData.getMeasurements()
for meas in measurements:
sampleFreq = siteData.getMeasurementSampleFreq(meas)
if sampleFreq not in options["sampleFreqs"]:
# don't need to calculate statistics for this sampling frequency
continue
projectText("Calculating stats for site {}, measurement {}".format(
site, meas))
# decimation parameters
decParams = getDecimationParameters(sampleFreq, projData.config)
decParams.printInfo()
numLevels = decParams.numLevels
# create the spectrum reader
specReader = SpectrumReader(
os.path.join(siteData.getMeasurementSpecPath(meas),
options["specdir"]))
# loop through decimation levels
for iDec in range(0, numLevels):
# open the spectra file for the current decimation level
check = specReader.openBinaryForReading("spectra", iDec)
if not check:
# probably because this decimation level not calculated
continue
specReader.printInfo()
# get windows
refTime = specReader.getReferenceTime()
winSize = specReader.getWindowSize()
winOlap = specReader.getWindowOverlap()
numWindows = specReader.getNumWindows()
evalFreq = decParams.getEvalFrequenciesForLevel(iDec)
sampleFreqDec = specReader.getSampleFreq()
globalOffset = specReader.getGlobalOffset()
fArray = specReader.getFrequencyArray()
statHandlers = {}
# create the statistic handlers
for stat in options["stats"]:
statElements = getStatElements(stat)
statHandlers[stat] = StatisticData(stat, refTime,
sampleFreqDec, winSize,
winOlap)
statHandlers[stat].setStatParams(numWindows, statElements,
evalFreq)
statHandlers[stat].comments = specReader.getComments()
statHandlers[stat].addComment(
projData.config.getConfigComment())
statHandlers[stat].addComment(
"Calculating statistic: {}".format(stat))
statHandlers[stat].addComment(
"Statistic components: {}".format(
listToString(statElements)))
# loop over windows and calculate the relevant statistics
for iW in range(0, numWindows):
# get data
winData = specReader.readBinaryWindowLocal(iW)
globalIndex = iW + globalOffset
# give the statistic calculator the spectra
statCalculator.setSpectra(fArray, winData, evalFreq)
# get the desired statistics
for sH in statHandlers:
data = statCalculator.getDataForStatName(sH)
statHandlers[sH].addStat(iW, globalIndex, data)
# save statistic
for sH in statHandlers:
statIO.setDatapath(
os.path.join(siteData.getMeasurementStatPath(meas),
options["specdir"]))
statIO.write(statHandlers[sH], iDec)
def calculateSpectra(projData: ProjectData, **kwargs) -> None:
"""Calculate spectra for the project time data
The philosophy is that spectra are calculated out for all data and later limited using statistics and time constraints
Parameters
----------
projData : ProjectData
A project data object
sites : str, List[str], optional
Either a single site or a list of sites
sampleFreqs : int, float, List[float], optional
The frequencies in Hz for which to calculate the spectra. Either a single frequency or a list of them.
chans : List[str], optional
The channels for which to calculate out the spectra
polreverse : Dict[str, bool]
Keys are channels and values are boolean flags for reversing
scale : Dict[str, float]
Keys are channels and values are floats to multiply the channel data by
calibrate : bool, optional
Flag whether to calibrate the data or not
notch : List[float], optional
List of frequencies to notch
filter : Dict, optional
Filter parameters
specdir : str, optional
The spectra directory to save the spectra data in
"""
# default options
options = {}
options["sites"] = projData.getSites()
options["sampleFreqs"]: List[float] = projData.getSampleFreqs()
options["chans"]: List[str] = []
options["polreverse"]: Union[bool, Dict[str, bool]] = False
options["scale"]: Union[bool, Dict[str, float]] = False
options["calibrate"]: bool = True
options["notch"]: List[float] = []
options["filter"]: Dict = {}
options["specdir"]: str = projData.config.configParams["Spectra"]["specdir"]
options = parseKeywords(options, kwargs)
# prepare calibrator
cal = getCalibrator(projData.calPath, projData.config)
if options["calibrate"]:
cal.printInfo()
datetimeRef = projData.refTime
for site in options["sites"]:
siteData = projData.getSiteData(site)
siteData.printInfo()
# calculate spectra for each frequency
for sampleFreq in options["sampleFreqs"]:
measurements = siteData.getMeasurements(sampleFreq)
projectText(
"Site {} has {:d} measurement(s) at sampling frequency {:.2f}".format(
site, len(measurements), sampleFreq
)
)
if len(measurements) == 0:
continue # no data files at this sample rate
for meas in measurements:
projectText(
"Calculating spectra for site {} and measurement {}".format(
site, meas
)
)
# get measurement start and end times - this is the time of the first and last sample
reader = siteData.getMeasurement(meas)
startTime = siteData.getMeasurementStart(meas)
stopTime = siteData.getMeasurementEnd(meas)
dataChans = (
options["chans"]
if len(options["chans"]) > 0
else reader.getChannels()
)
timeData = reader.getPhysicalData(startTime, stopTime, chans=dataChans)
timeData.addComment(breakComment())
timeData.addComment("Calculating project spectra")
timeData.addComment(projData.config.getConfigComment())
# apply various options
applyPolarisationReversalOptions(options, timeData)
applyScaleOptions(options, timeData)
applyCalibrationOptions(options, cal, timeData, reader)
applyFilterOptions(options, timeData)
applyNotchOptions(options, timeData)
# define decimation and window parameters
decParams = getDecimationParameters(sampleFreq, projData.config)
decParams.printInfo()
numLevels = decParams.numLevels
winParams = getWindowParameters(decParams, projData.config)
dec = Decimator(timeData, decParams)
timeData.addComment(
"Decimating with {} levels and {} frequencies per level".format(
numLevels, decParams.freqPerLevel
)
)
# loop through decimation levels
for iDec in range(0, numLevels):
# get the data for the current level
check = dec.incrementLevel()
if not check:
break # not enough data
timeData = dec.timeData
# create the windower and give it window parameters for current level
sampleFreqDec = dec.sampleFreq
win = Windower(
datetimeRef,
timeData,
winParams.getWindowSize(iDec),
winParams.getOverlap(iDec),
)
if win.numWindows < 2:
break # do no more decimation
# add some comments
timeData.addComment(
"Evaluation frequencies for this level {}".format(
listToString(decParams.getEvalFrequenciesForLevel(iDec))
)
)
timeData.addComment(
"Windowing with window size {} samples and overlap {} samples".format(
winParams.getWindowSize(iDec), winParams.getOverlap(iDec)
)
)
# create the spectrum calculator and statistics calculators
specCalc = SpectrumCalculator(
sampleFreqDec, winParams.getWindowSize(iDec)
)
# get ready a file to save the spectra
specPath = os.path.join(
siteData.getMeasurementSpecPath(meas), options["specdir"]
)
specWrite = SpectrumWriter(specPath, datetimeRef)
specWrite.openBinaryForWriting(
"spectra",
iDec,
sampleFreqDec,
winParams.getWindowSize(iDec),
winParams.getOverlap(iDec),
win.winOffset,
win.numWindows,
dataChans,
)
# loop though windows, calculate spectra and save
for iW in range(0, win.numWindows):
# get the window data
winData = win.getData(iW)
# calculate spectra
specData = specCalc.calcFourierCoeff(winData)
# write out spectra
specWrite.writeBinary(specData)
# close spectra file
specWrite.writeCommentsFile(timeData.getComments())
specWrite.closeFile()