def getPhysicalSamples(self, **kwargs):
"""Get ascii data scaled to physical values
Warnings
--------
No scaling happens in getPhysicalSamples. Ascii data is assumed to be properly scaled to mV for magnetic channels and mV/km for electric channels (i.e. field units)
Parameters
----------
chans : List[str]
List of channels to return if not all are required
startSample : int
First sample to return
endSample : int
Last sample to return
remaverage : bool
Remove average from the data
remzeros : bool
Remove zeroes from the data
remnans: bool
Remove NanNs from the data
Returns
-------
TimeData
Time data object
"""
options = self.parseGetDataKeywords(kwargs)
timeData = self.getUnscaledSamples(
chans=options["chans"],
startSample=options["startSample"],
endSample=options["endSample"],
)
# no further scaling applied to ascii data
for chan in options["chans"]:
# if remove zeros - False by default
if options["remzeros"]:
timeData.data[chan] = removeZerosSingle(timeData.data[chan])
# if remove nans - False by default
if options["remnans"]:
timeData.data[chan] = removeNansSingle(timeData.data[chan])
# remove the average from the data - True by default
# do this after all scaling and removing nans and zeros
if options["remaverage"]:
timeData.data[chan] = timeData.data[chan] - np.average(
timeData.data[chan]
)
timeData.addComment(
"Remove zeros: {}, remove nans: {}, remove average: {}".format(
options["remzeros"], options["remnans"], options["remaverage"]
)
)
return timeData
def getPhysicalSamples(self, **kwargs) -> TimeData:
"""Get data scaled to physical values
Parameters
----------
chans : List[str]
List of channels to return if not all are required
startSample : int
First sample to return
endSample : int
Last sample to return
remaverage : bool
Remove average from the data
remzeros : bool
Remove zeroes from the data
remnans: bool
Remove NanNs from the data
Returns
-------
TimeData
Time data object
"""
options = self.parseGetDataKeywords(kwargs)
# get data
timeData = self.getUnscaledSamples(
chans=options["chans"],
startSample=options["startSample"],
endSample=options["endSample"],
)
# need to remove the gain
for chan in options["chans"]:
# remove the gain
timeData.data[
chan] = 1.0 * timeData.data[chan] / self.getChanGain1(chan)
timeData.addComment(
"Scaling channel {} with scalar {} to give mV".format(
chan, 1.0 / self.getChanGain1(chan)))
# divide by distance in km
if chan == "Ex":
# multiply by 1000/self.getChanDx same as dividing by dist in km
timeData.data[
chan] = 1000 * timeData.data[chan] / self.getChanDx(chan)
timeData.addComment(
"Dividing channel {} by electrode distance {} km to give mV/km"
.format(chan,
self.getChanDx(chan) / 1000.0))
if chan == "Ey":
# multiply by 1000/self.getChanDy same as dividing by dist in km
timeData.data[
chan] = 1000 * timeData.data[chan] / self.getChanDy(chan)
timeData.addComment(
"Dividing channel {} by electrode distance {} km to give mV/km"
.format(chan,
self.getChanDy(chan) / 1000.0))
# if remove zeros - False by default
if options["remzeros"]:
timeData.data[chan] = removeZerosSingle(timeData.data[chan])
# if remove nans - False by default
if options["remnans"]:
timeData.data[chan] = removeNansSingle(timeData.data[chan])
# remove the average from the data - True by default
if options["remaverage"]:
timeData.data[chan] = timeData.data[chan] - np.average(
timeData.data[chan])
# add comments
timeData.addComment(
"The required Phoneix scaling to field units is still unverified. This is experimental and use cautiously."
)
timeData.addComment(
"Remove zeros: {}, remove nans: {}, remove average: {}".format(
options["remzeros"], options["remnans"],
options["remaverage"]))
return timeData
def getPhysicalSamples(self, **kwargs):
r"""Get data scaled to physical values
Depending on the data format, the scalings required to convert to field physical units is different. The method in the base DataReader class covers ATS and internal file format.
resistics will always provide physical samples in field units. That means
- Electrical channels in mV/km
- Magnetic channels in mV
- To get magnetic fields in nT, calibration needs to be performed
If the channel header scaling_applied is set to True, no scaling of the unscaled data is done. This is to cover the internal data format where all scalings may already have been applied.
Notes
-----
The raw data units for ATS data are in counts. To get data in field units, ATS data is first multipled by the least significat bit (lsb) defined in the header files,
.. math::
data = data * lsb,
giving data in mV. The lsb includes the gain removal, so no separate gain removal needs to be performed.
For electrical channels, there is additional step of dividing by the electrode spacing, which is provided in metres. The extra factor of a 1000 is to convert this to km to give mV/km for electric channels
.. math::
data = \frac{1000 * data}{electrodeSpacing}
Finally, to get magnetic channels in nT, the magnetic channels need to be calibrated.
Parameters
----------
chans : List[str]
List of channels to return if not all are required
startSample : int
First sample to return
endSample : int
Last sample to return
remaverage : bool
Remove average from the data
remzeros : bool
Remove zeroes from the data
remnans: bool
Remove NanNs from the data
Returns
-------
TimeData
Time data object
"""
options = self.parseGetDataKeywords(kwargs)
timeData = self.getUnscaledSamples(
chans=options["chans"],
startSample=options["startSample"],
endSample=options["endSample"],
)
# multiply each chan by least significant bit of chan
for chan in options["chans"]:
if not self.getChanScalingApplied(chan):
# apply LSB to give data in mV
timeData.data[chan] = timeData.data[chan] * self.getChanLSB(
chan)
timeData.addComment(
"Scaling channel {} with scalar {} to give mV".format(
chan, self.getChanLSB(chan)))
# divide by the distance - this should only be for the electric channels
# again, this might already be applied
if chan == "Ex":
# multiply by 1000/self.getChanDx same as dividing by dist in km
timeData.data[chan] = (1000 * timeData.data[chan] /
self.getChanDx(chan))
timeData.addComment(
"Dividing channel {} by electrode distance {} km to give mV/km"
.format(chan,
self.getChanDx(chan) / 1000.0))
if chan == "Ey":
# multiply by 1000/self.getChanDy same as dividing by dist in km
timeData.data[chan] = (1000 * timeData.data[chan] /
self.getChanDy(chan))
timeData.addComment(
"Dividing channel {} by electrode distance {} km to give mV/km"
.format(chan,
self.getChanDy(chan) / 1000.0))
# if remove zeros - False by default
if options["remzeros"]:
timeData.data[chan] = removeZerosSingle(timeData.data[chan])
# if remove nans - False by default
if options["remnans"]:
timeData.data[chan] = removeNansSingle(timeData.data[chan])
# remove the average from the data - True by default
# do this after all scaling and removing nans and zeros
if options["remaverage"]:
timeData.data[chan] = timeData.data[chan] - np.average(
timeData.data[chan])
timeData.addComment(
"Remove zeros: {}, remove nans: {}, remove average: {}".format(
options["remzeros"], options["remnans"],
options["remaverage"]))
return timeData
def getPhysicalSamples(self, **kwargs):
"""Get data scaled to physical values
resistics uses field units, meaning physical samples will return the following:
- Electrical channels in mV/km
- Magnetic channels in mV
- To get magnetic fields in nT, calibration needs to be performed
Notes
-----
The method getUnscaledSamples multiplies the raw data by the ts_lsb converting it to mV. Because gain is removed when getting the unscaledSamples and all channel data is in mV, the only calculation that has to be done is to divide by the dipole lengths (east-west spacing and north-south spacing).
To get magnetic fields in nT, they have to be calibrated.
Parameters
----------
chans : List[str]
List of channels to return if not all are required
startSample : int
First sample to return
endSample : int
Last sample to return
remaverage : bool
Remove average from the data
remzeros : bool
Remove zeroes from the data
remnans: bool
Remove NanNs from the data
Returns
-------
TimeData
Time data object
"""
# initialise chans, startSample and endSample with the whole dataset
options = self.parseGetDataKeywords(kwargs)
# get data
timeData = self.getUnscaledSamples(
chans=options["chans"],
startSample=options["startSample"],
endSample=options["endSample"],
)
# Scalars are applied in getUnscaledSamples to convert to mV - this is for ease of calculation and because each data file in the run might have a separate scaling
# all that is left is to divide by the dipole length in km and remove the average
for chan in options["chans"]:
if chan == "Ex":
# multiply by 1000/self.getChanDx same as dividing by dist in km
timeData.data[
chan] = 1000 * timeData.data[chan] / self.getChanDx(chan)
timeData.addComment(
"Dividing channel {} by electrode distance {} km to give mV/km"
.format(chan,
self.getChanDx(chan) / 1000.0))
if chan == "Ey":
# multiply by 1000/self.getChanDy same as dividing by dist in km
timeData.data[
chan] = 1000 * timeData.data[chan] / self.getChanDy(chan)
timeData.addComment(
"Dividing channel {} by electrode distance {} km to give mV/km"
.format(chan,
self.getChanDy(chan) / 1000.0))
# if remove zeros - False by default
if options["remzeros"]:
timeData.data[chan] = removeZerosSingle(timeData.data[chan])
# if remove nans - False by default
if options["remnans"]:
timeData.data[chan] = removeNansSingle(timeData.data[chan])
# remove the average from the data - True by default
if options["remaverage"]:
timeData.data[chan] = timeData.data[chan] - np.average(
timeData.data[chan])
# add comments
timeData.addComment(
"Remove zeros: {}, remove nans: {}, remove average: {}".format(
options["remzeros"], options["remnans"],
options["remaverage"]))
return timeData
def getPhysicalSamples(self, **kwargs):
"""Get data scaled to physical values
resistics uses field units, meaning physical samples will return the following:
- Electrical channels in mV/km
- Magnetic channels in mV
- To get magnetic fields in nT, calibration needs to be performed
Notes
-----
Once Lemi B423 data is scaled (which optionally happens in getUnscaledSamples), the magnetic channels is in mV with gain applied and the electric channels is uV (microvolts). Therefore:
- Electric channels need to divided by 1000 along with dipole length division in km (east-west spacing and north-south spacing) to return mV/km.
- Magnetic channels need to be divided by the internal gain value which should be set in the headers
To get magnetic fields in nT, they have to be calibrated.
Parameters
----------
chans : List[str]
List of channels to return if not all are required
startSample : int
First sample to return
endSample : int
Last sample to return
remaverage : bool
Remove average from the data
remzeros : bool
Remove zeroes from the data
remnans: bool
Remove NanNs from the data
Returns
-------
TimeData
Time data object
"""
# initialise chans, startSample and endSample with the whole dataset
options = self.parseGetDataKeywords(kwargs)
# get unscaled data but with gain scalings applied
timeData = self.getUnscaledSamples(
chans=options["chans"],
startSample=options["startSample"],
endSample=options["endSample"],
scale=True,
)
# convert to field units and divide by dipole lengths
for chan in options["chans"]:
if isElectric(chan):
timeData.data[chan] = timeData.data[chan] / 1000.0
timeData.addComment(
"Dividing channel {} by 1000 to convert microvolt to millivolt"
.format(chan))
if isMagnetic(chan):
timeData.data[chan] = timeData.data[chan] / self.getChanGain1(
chan)
timeData.addComment(
"Removing gain of {} from channel {}".format(
self.getChanGain1(chan), chan))
if chan == "Ex":
# multiply by 1000/self.getChanDx same as dividing by dist in km
timeData.data[chan] = (1000.0 * timeData.data[chan] /
self.getChanDx(chan))
timeData.addComment(
"Dividing channel {} by electrode distance {} km to give mV/km"
.format(chan,
self.getChanDx(chan) / 1000.0))
if chan == "Ey":
# multiply by 1000/self.getChanDy same as dividing by dist in km
timeData.data[
chan] = 1000 * timeData.data[chan] / self.getChanDy(chan)
timeData.addComment(
"Dividing channel {} by electrode distance {:.6f} km to give mV/km"
.format(chan,
self.getChanDy(chan) / 1000.0))
# if remove zeros - False by default
if options["remzeros"]:
timeData.data[chan] = removeZerosSingle(timeData.data[chan])
# if remove nans - False by default
if options["remnans"]:
timeData.data[chan] = removeNansSingle(timeData.data[chan])
# remove the average from the data - True by default
if options["remaverage"]:
timeData.data[chan] = timeData.data[chan] - np.average(
timeData.data[chan])
# add comments
timeData.addComment(
"Remove zeros: {}, remove nans: {}, remove average: {}".format(
options["remzeros"], options["remnans"],
options["remaverage"]))
return timeData