def getDataReader(datapath: str) -> Union[DataReader, bool]:
"""Get the data reader for a time data directory (and format)
Format is determined through a mix of header and data file extensions. Not perfect, but it saves the user from having to specify each time
Parameters
----------
datapath : str
Path to data directory
Returns
-------
dataReader : DataReader (DataReaderATS, DataReaderSPAM, DataReaderInternal, DataReaderPhonix, DataReaderLemiB423, DataReaderLemiB423E)
Data reader object
"""
# Lemi B423 data files
headerF = glob.glob(os.path.join(datapath, "*.h423"))
if len(headerF) > 0: # then lemi b423 format
return DataReaderLemiB423(datapath)
# Lemi B423E data files
headerF = glob.glob(os.path.join(datapath, "*.h423E"))
if len(headerF) > 0: # then lemi b423 format
return DataReaderLemiB423E(datapath)
# ATS data files
headerF = glob.glob(os.path.join(datapath, "*.xml"))
headerF = headerF + glob.glob(os.path.join(datapath, "*.XML"))
if len(headerF) > 0: # then ats format
return DataReaderATS(datapath)
# SPAM data files
headerF = glob.glob(os.path.join(datapath, "*.xtrx"))
headerF = headerF + glob.glob(os.path.join(datapath, "*.XTRX"))
if len(headerF) > 0: # then xtrx format
return DataReaderSPAM(datapath)
headerF = glob.glob(os.path.join(datapath, "*.xtr"))
headerF = headerF + glob.glob(os.path.join(datapath, "*.XTR"))
if len(headerF) > 0: # then xtr format
return DataReaderSPAM(datapath)
# Phoenix data files
headerF = glob.glob(os.path.join(datapath, "*.tbl"))
headerF = headerF + glob.glob(os.path.join(datapath, "*.TBL"))
if len(headerF) > 0: # then xtr format
return DataReaderPhoenix(datapath)
# internal header format
headerF = glob.glob(os.path.join(datapath, "*.hdr"))
headerF = headerF + glob.glob(os.path.join(datapath, "*.HDR"))
if len(headerF) > 0: # then internal format or ascii format
dataF = glob.glob(os.path.join(datapath, "*.ascii"))
if len(dataF) > 0:
return DataReaderAscii(datapath)
# otherwise assume internal format
return DataReaderInternal(datapath)
# if nothing found, return false
return False
fig.tight_layout(rect=[0, 0.02, 1, 0.96])
plt.show()
fig.savefig(os.path.join("images", "ascii.png"))
# now write out as internal format
from resistics.ioHandlers.dataWriterInternal import DataWriterInternal
ascii_2intenrnal = os.path.join("timeData", "asciiInternal")
writer = DataWriterInternal()
writer.setOutPath(ascii_2intenrnal)
writer.writeDataset(asciiReader, physical=True)
# read in internal format
from resistics.ioHandlers.dataReaderInternal import DataReaderInternal
internalReader = DataReaderInternal(ascii_2intenrnal)
internalReader.printInfo()
internalReader.printComments()
internalData = internalReader.getPhysicalSamples()
internalData.printInfo()
# now plot the two datasets together
fig = plt.figure(figsize=(16, 3 * asciiData.numChans))
asciiData.view(fig=fig, sampleStop=500, label="ASCII format", legend=True)
internalData.view(fig=fig,
sampleStop=500,
label="Internal format",
legend=True)
fig.tight_layout(rect=[0, 0.02, 1, 0.96])
plt.show()
fig.savefig(os.path.join("images", "ascii_vs_internal.png"))
fig.tight_layout(rect=[0, 0.02, 1, 0.96])
plt.show()
fig.savefig(os.path.join("images", "spam.png"))
# write out as the internal format
from resistics.ioHandlers.dataWriterInternal import DataWriterInternal
spam_2internal = os.path.join("timeData", "spamInternal")
writer = DataWriterInternal()
writer.setOutPath(spam_2internal)
writer.writeDataset(spamReader, physical=True)
# read in the internal format dataset and see what's in the comments
from resistics.ioHandlers.dataReaderInternal import DataReaderInternal
internalReader = DataReaderInternal(spam_2internal)
internalReader.printInfo()
internalReader.printComments()
physicalInternalData = internalReader.getPhysicalData(startTime, stopTime)
physicalInternalData.printInfo()
# now plot the two datasets together
fig = plt.figure(figsize=(16, 3 * physicalSPAMData.numChans))
physicalSPAMData.view(fig=fig, sampleStop=500, label="SPAM format")
physicalInternalData.view(fig=fig,
sampleStop=500,
label="Internal format",
legend=True)
fig.tight_layout(rect=[0, 0.02, 1, 0.96])
plt.show()
fig.savefig(os.path.join("images", "spam_vs_internal.png"))
fig = viewTime(
proj,
"2012-02-10 11:05:00",
"2012-02-10 11:14:00",
sites=["site1", "site1_gaps"],
filter={"lpfilt": 16},
chans=["Ex", "Hy"],
show=False,
plotoptions=plotOptions,
)
fig.savefig(Path(proj.imagePath, "viewTimeGaps.png"))
from resistics.ioHandlers.dataReaderInternal import DataReaderInternal
siteGaps = proj.getSiteData("site1_gaps")
readerSection1 = DataReaderInternal(
siteGaps.getMeasurementTimePath("meas_2012-02-10_11-05-00_section1"))
timeData1 = readerSection1.getPhysicalSamples(remaverage=False)
timeData1.printInfo()
readerSection2 = DataReaderInternal(
siteGaps.getMeasurementTimePath("meas_2012-02-10_11-05-00_section2"))
timeData2 = readerSection2.getPhysicalSamples(remaverage=False)
timeData2.printInfo()
from resistics.utilities.utilsInterp import fillGap
timeDataFilled = fillGap(timeData1, timeData2)
timeDataFilled.printInfo()
samplesToView = 14 * 60 * 4096
fig = timeDataFilled.view(sampleStop=samplesToView, chans=["Ex", "Hy"])
fig.savefig(Path(proj.imagePath, "timeDataFilled.png"))
headers = spamReader.getHeaders()
chanHeaders, chanMap = spamReader.getChanHeaders()
writer = DataWriterInternal()
writer.setOutPath(interpPath)
writer.writeData(
headers,
chanHeaders,
interpData,
physical=True,
)
writer.printInfo()
# read in the internal data
from resistics.ioHandlers.dataReaderInternal import DataReaderInternal
interpReader = DataReaderInternal(interpPath)
interpReader.printInfo()
interpReader.printComments()
# get data between a time range
startTime = "2016-02-07 02:10:00"
stopTime = "2016-02-07 02:30:00"
spamData = spamReader.getPhysicalData(startTime, stopTime)
interpData = interpReader.getPhysicalData(startTime, stopTime)
# plot the datasets
import matplotlib.pyplot as plt
fig = plt.figure(figsize=(12, 8))
plt.plot(spamData.getDateArray()[0:100], spamData.data["Ex"][0:100], "o--")
plt.plot(interpData.getDateArray()[0:100], interpData.data["Ex"][0:100], "x:")
filteredData = highPass(physicalData, 4, inplace=False)
fig = plt.figure(figsize=(16, 3 * physicalData.numChans))
filteredData.view(fig=fig, sampleEnd=20000)
fig.tight_layout(rect=[0, 0.02, 1, 0.96])
plt.show()
fig.savefig(os.path.join("images", "phoenixFiltered.png"))
# reformat the continuous sampling frequency
phoenix_2internal = os.path.join("timeData", "phoenixInternal")
phoenixReader.reformatContinuous(phoenix_2internal)
# reading output
from resistics.ioHandlers.dataReaderInternal import DataReaderInternal
internalReader = DataReaderInternal(
os.path.join(phoenix_2internal,
"meas_ts5_2011-11-13-17-04-02_2011-11-14-14-29-46"))
internalReader.printInfo()
internalReader.printComments()
# read in physical data
internalData = internalReader.getPhysicalData(startTime, stopTime)
# plot the two together
fig = plt.figure(figsize=(16, 3 * physicalData.numChans))
physicalData.view(fig=fig, label="Phoenix format data")
internalData.view(fig=fig, label="Internal format data", legend=True)
fig.tight_layout(rect=[0, 0.02, 1, 0.96])
plt.show()
fig.savefig(os.path.join("images", "phoenix_vs_internal_continuous.png"))
fig.tight_layout(rect=[0, 0.02, 1, 0.96])
plt.show()
fig.savefig(os.path.join("images", "ats_filteredData.png"))
# now write out as internal format
from resistics.ioHandlers.dataWriterInternal import DataWriterInternal
ats_2intenrnal = os.path.join("timeData", "atsInternal")
writer = DataWriterInternal()
writer.setOutPath(ats_2intenrnal)
writer.writeDataset(atsReader, physical=True)
# read in internal format
from resistics.ioHandlers.dataReaderInternal import DataReaderInternal
internalReader = DataReaderInternal(ats_2intenrnal)
internalReader.printInfo()
internalReader.printComments()
physicalInternalData = internalReader.getPhysicalData(startTime, stopTime)
physicalInternalData.printInfo()
# now plot the two datasets together
fig = plt.figure(figsize=(16, 3 * physicalATSData.numChans))
physicalATSData.view(fig=fig, sampleStop=200, label="ATS format", legend=True)
physicalInternalData.view(fig=fig, sampleStop=200, label="Internal format", legend=True)
fig.tight_layout(rect=[0, 0.02, 1, 0.96])
plt.show()
fig.savefig(os.path.join("images", "ats_vs_internal.png"))
# now write out as ascii format
from resistics.ioHandlers.dataWriterAscii import DataWriterAscii
def readHeader(self) -> None:
"""Read the B423 measurement file headers"""
DataReaderInternal.readHeader(self)
self.readMeasParams()
import os
from resistics.ioHandlers.dataReaderInternal import DataReaderInternal
# data paths
internalPath = os.path.join("timeData", "atsInternal")
internalReader = DataReaderInternal(internalPath)
internalReader.printInfo()
# get data
internalData = internalReader.getPhysicalSamples(startSample=0, endSample=20000)
internalData.printInfo()
# plot
import matplotlib.pyplot as plt
fig = plt.figure(figsize=(16, 3 * internalData.numChans))
internalData.view(fig=fig, sampleStart=0, sampleStop=1000)
fig.tight_layout(rect=[0, 0.02, 1, 0.96])
plt.show()
fig.savefig(os.path.join("images", "internalData.png"))
# get the data file for each channel
channels = internalData.chans
chan2File = dict()
for chan in channels:
chan2File[chan] = internalReader.getChanDataFile(chan)
# read in the Ex data using numpy
import numpy as np
dataFile = os.path.join(internalPath, chan2File["Ex"])