def test_isElectric() -> None: from resistics.common.checks import isElectric assert isElectric("Ex") assert isElectric("Ey") assert not isElectric("Hx") assert not isElectric("Hy") assert not isElectric("Hz")
def getChanChopper(self, chan) -> bool: """Get channel chopper The chopper is an amplifier present in some instruments. There might be different calibration files for chopper on or off. Returns ------- bool Flag designating whether chopper is on or off """ echopper = self.getChanHeader(chan, "echopper") hchopper = self.getChanHeader(chan, "hchopper") # return true if the chopper amplifier was on if isElectric(chan) and echopper: return True if isMagnetic(chan) and hchopper: return True return False
def view(self, **kwargs) -> Figure: """View timeseries data as a line plot Parameters ---------- sampleStart : int, optional Sample to start plotting from sampleStop : int, optional Sample to plot to fig : matplotlib.pyplot.figure, optional A figure object plotfonts : Dict, optional A dictionary of plot fonts chans : List[str] Channels to plot label : str, optional Label for the plots xlim : List, optional Limits for the x axis legened : bool Boolean flag for adding a legend Returns ------- plt.figure Matplotlib figure object """ # the number of samples to plot sampleStart = 0 sampleStop = 4096 if "sampleStart" in kwargs: sampleStart = kwargs["sampleStart"] if "sampleStop" in kwargs: sampleStop = kwargs["sampleStop"] if sampleStop >= self.numSamples: sampleStop = self.numSamples - 1 # get the x axis ready x = self.getDateArray() start = x[sampleStart] stop = x[sampleStop] # now plot fig = (plt.figure(kwargs["fig"].number) if "fig" in kwargs else plt.figure(figsize=(20, 2 * self.numChans))) plotfonts = kwargs[ "plotfonts"] if "plotfonts" in kwargs else getViewFonts() # suptitle st = fig.suptitle( "Time data from {} to {}, samples {} to {}".format( start, stop, sampleStart, sampleStop), fontsize=plotfonts["suptitle"], ) st.set_y(0.98) # now plot the data dataChans = kwargs["chans"] if "chans" in kwargs else self.chans numDataChans = len(dataChans) for idx, chan in enumerate(dataChans): ax = plt.subplot(numDataChans, 1, idx + 1) plt.title("Channel {}".format(chan), fontsize=plotfonts["title"]) # check if channel exists in data and if not, leave empty plot so it's clear if chan not in self.data: continue # label for plot lab = (kwargs["label"] if "label" in kwargs else "{}: {} to {}".format(chan, start, stop)) # plot data plt.plot( x[sampleStart:sampleStop + 1], self.data[chan][sampleStart:sampleStop + 1], label=lab, ) # add time label if idx == numDataChans - 1: plt.xlabel("Time", fontsize=plotfonts["axisLabel"]) # set the xlim xlim = kwargs["xlim"] if "xlim" in kwargs else [start, stop] plt.xlim(xlim) # y axis options if isElectric(chan): plt.ylabel("mV/km", fontsize=plotfonts["axisLabel"]) else: plt.ylabel("nT or mV", fontsize=plotfonts["axisLabel"]) plt.grid(True) # set tick sizes for label in ax.get_xticklabels() + ax.get_yticklabels(): label.set_fontsize(plotfonts["axisTicks"]) # legend if "legend" in kwargs and kwargs["legend"]: plt.legend(loc=4) # show if the figure is not in keywords if "fig" not in kwargs: plt.tight_layout(rect=[0, 0.02, 1, 0.96]) plt.show() return fig
def headersFromTable(self, tableData: Dict) -> Tuple[Dict, List]: """Populate the headers from the table values Parameters ---------- tableData : OrderedDictDict Ordered dictionary with table data Returns ------- headers : Dict Dictionary of general headers chanHeaders : Dict List of channel headers """ # initialise storage headers = {} chanHeaders = [] # get the sample freqs for each ts file self.tsSampleFreqs = [] for tsNum in self.tsNums: self.tsSampleFreqs.append(tableData["SRL{}".format(tsNum)]) # for sample frequency, use the continuous channel headers["sample_freq"] = self.tsSampleFreqs[self.continuousI] # these are the unix time stamps firstDate, firstTime, lastDate, lastTime = self.getDates(tableData) # the start date is equal to the time of the first record headers["start_date"] = firstDate headers["start_time"] = firstTime datetimeStart = datetime.strptime("{} {}".format(firstDate, firstTime), "%Y-%m-%d %H:%M:%S.%f") # the stop date datetimeLast = datetime.strptime("{} {}".format(lastDate, lastTime), "%Y-%m-%d %H:%M:%S.%f") # records are usually equal to one second (beginning on 0 and ending on the last sample before the next 0) datetimeStop = datetimeLast + timedelta( seconds=(1.0 - 1.0 / headers["sample_freq"])) # put the stop date and time in the headers headers["stop_date"] = datetimeStop.strftime("%Y-%m-%d") headers["stop_time"] = datetimeStop.strftime("%H:%M:%S.%f") # here calculate number of samples deltaSeconds = (datetimeStop - datetimeStart).total_seconds() # calculate number of samples - have to add one because the time given in SPAM recording is the actual time of the last sample numSamples = round(deltaSeconds * headers["sample_freq"]) + 1 headers["num_samples"] = numSamples headers["ats_data_file"] = self.continuousF # deal with the channel headers # now want to do this in the correct order # chan headers should reflect the order in the data chans = ["Ex", "Ey", "Hx", "Hy", "Hz"] chanOrder = [] for chan in chans: chanOrder.append(tableData["CH{}".format(chan.upper())]) # sort the lists in the right order based on chanOrder chanOrder, chans = (list(x) for x in zip( *sorted(zip(chanOrder, chans), key=lambda pair: pair[0]))) for chan in chans: chanH = self.chanDefaults() # set the sample frequency from the main headers chanH["sample_freq"] = headers["sample_freq"] # channel output information (sensor_type, channel_type, ts_lsb, pos_x1, pos_x2, pos_y1, pos_y2, pos_z1, pos_z2, sensor_sernum) chanH["ats_data_file"] = self.dataF[self.continuousI] chanH["num_samples"] = numSamples # channel information chanH["channel_type"] = consistentChans( chan) # consistent chan naming # magnetic channels only if isMagnetic(chanH["channel_type"]): chanH["sensor_sernum"] = tableData["{}SN".format( chan.upper())][-4:] chanH["sensor_type"] = "Phoenix" # channel input information (gain_stage1, gain_stage2, hchopper, echopper) chanH["gain_stage1"] = tableData["HGN"] chanH["gain_stage2"] = 1 # electric channels only if isElectric(chanH["channel_type"]): # the distances if chan == "Ex": chanH["pos_x1"] = float(tableData["EXLN"]) / 2.0 chanH["pos_x2"] = chanH["pos_x1"] if chan == "Ey": chanH["pos_y1"] = float(tableData["EYLN"]) / 2.0 chanH["pos_y2"] = chanH["pos_y1"] # channel input information (gain_stage1, gain_stage2, hchopper, echopper) chanH["gain_stage1"] = tableData["EGN"] chanH["gain_stage2"] = 1 # append chanHeaders to the list chanHeaders.append(chanH) # data information (meas_channels, sample_freq) headers["meas_channels"] = len( chans) # this gets reformatted to an int later # return the headers and chanHeaders from this file return headers, chanHeaders
def viewTime(projData: ProjectData, startDate: str, endDate: str, **kwargs) -> Union[Figure, None]: """View timeseries in the project Parameters ---------- projData : ProjectData The project data instance startDate : str The start of the data range to plot endDate : str The end of the date range to plot sites : List[str], optional List of sites sampleFreqs : List[float], optional List of sample frequencies to plot chans : List[str], optional List of channels to plot polreverse : Dict[str, bool] Keys are channels and values are boolean flags for reversing scale : Dict[str, float] Keys are channels and values are floats to multiply the channel data by calibrate : bool, optional Boolean flag to calibrate data normalise : bool, optional Boolean flag to normalise the data. Default is False and setting to True will normalise each channel independently. notch : List[float], optional List of frequencies to notch out filter : Dict, optional Filter parameters show : bool, optional Boolean flag to show the plot save : bool, optional Boolean flag to save the plot to images folder plotoptions : Dict Dictionary of plot options Returns ------- matplotlib.pyplot.figure or None A matplotlib figure unless the plot is not shown and is saved, in which case None and the figure is closed. """ from resistics.project.shortcuts import getCalibrator from resistics.project.preprocess import ( applyPolarisationReversalOptions, applyScaleOptions, applyCalibrationOptions, applyFilterOptions, applyNormaliseOptions, applyNotchOptions, ) from resistics.common.plot import savePlot, plotOptionsTime options = {} options["sites"]: List[str] = projData.sites options["sampleFreqs"]: Union[List[float], List[str]] = projData.getSampleFreqs() options["chans"]: List[str] = ["Ex", "Ey", "Hx", "Hy", "Hz"] options["polreverse"]: Union[bool, Dict[str, bool]] = False options["scale"]: Union[bool, Dict[str, float]] = False options["calibrate"]: bool = False options["normalise"]: bool = False options["filter"]: Dict = {} options["notch"]: List[float] = [] options["show"]: bool = True options["save"]: bool = False options["plotoptions"]: Dict = plotOptionsTime() options = parseKeywords(options, kwargs) # prepare calibrator cal = getCalibrator(projData.calPath, projData.config) if options["calibrate"]: cal.printInfo() # format startDate and endDate start = datetime.strptime("{}.000".format(startDate), "%Y-%m-%d %H:%M:%S.%f") stop = datetime.strptime("{}.000".format(endDate), "%Y-%m-%d %H:%M:%S.%f") # collect relevant data - dictionary to store timeData timeDataAll = {} for site in options["sites"]: siteData = projData.getSiteData(site) if isinstance(siteData, bool): # site does not exist continue siteData.printInfo() measurements = siteData.getMeasurements() timeDataAll[site] = {} # loop over measurements and save data for each one for meas in measurements: sampleFreq = siteData.getMeasurementSampleFreq(meas) if sampleFreq not in options["sampleFreqs"]: continue # check if data contributes to user defined time period siteStart = siteData.getMeasurementStart(meas) siteStop = siteData.getMeasurementEnd(meas) if siteStop < start or siteStart > stop: continue reader = siteData.getMeasurement(meas) # get the samples of the datetimes sampleStart, sampleStop = reader.time2sample(start, stop) # as the samples returned from time2sample are rounded use sample2time to get the appropriate start and end times for those samples readStart, readStop = reader.sample2time(sampleStart, sampleStop) # get the data for any available channels meaning even those sites with missing channels can be plotted timeData = reader.getPhysicalData(readStart, readStop) projectText( "Plotting measurement {} of site {} between {} and {}".format( meas, site, readStart, readStop)) # apply various options applyPolarisationReversalOptions(options, timeData) applyScaleOptions(options, timeData) applyCalibrationOptions(options, cal, timeData, reader) applyFilterOptions(options, timeData) applyNotchOptions(options, timeData) applyNormaliseOptions(options, timeData) timeDataAll[site][meas] = timeData # plot all the data plotfonts = options["plotoptions"]["plotfonts"] fig = plt.figure(figsize=options["plotoptions"]["figsize"]) for site in timeDataAll: for meas in timeDataAll[site]: timeData = timeDataAll[site][meas] timeData.view( sampleStop=timeDataAll[site][meas].numSamples - 1, fig=fig, chans=options["chans"], label="{} - {}".format(site, meas), xlim=[start, stop], plotfonts=plotfonts, ) # add the suptitle st = fig.suptitle( "Time data from {} to {}".format(start.strftime("%Y-%m-%d %H-%M-%S"), stop.strftime("%Y-%m-%d %H-%M-%S")), fontsize=plotfonts["suptitle"], ) st.set_y(0.98) # do the axis labels numChans = len(options["chans"]) for idx, chan in enumerate(options["chans"]): plt.subplot(numChans, 1, idx + 1) # do the yaxis if isElectric(chan): plt.ylabel("mV/km", fontsize=plotfonts["axisLabel"]) if len(options["plotoptions"]["Eylim"]) > 0: plt.ylim(options["plotoptions"]["Eylim"]) else: if options["calibrate"]: plt.ylabel("nT", fontsize=plotfonts["axisLabel"]) else: plt.ylabel("mV", fontsize=plotfonts["axisLabel"]) if len(options["plotoptions"]["Hylim"]) > 0: plt.ylim(options["plotoptions"]["Hylim"]) plt.legend(loc=1, fontsize=plotfonts["legend"]) # plot format fig.tight_layout(rect=[0, 0.02, 1, 0.96]) fig.subplots_adjust(top=0.92) # plot show and save if options["save"]: impath = projData.imagePath filename = "timeData_{}_{}".format( start.strftime("%Y-%m-%d_%H-%M-%S_"), stop.strftime("%Y-%m-%d_%H-%M-%S")) savename = savePlot(impath, filename, fig) projectText("Image saved to file {}".format(savename)) if options["show"]: plt.show(block=options["plotoptions"]["block"]) if not options["show"] and options["save"]: plt.close(fig) return None return fig
def view(self, **kwargs) -> Figure: """Plot spectra data Parameters ---------- fig : matplotlib.pyplot.figure, optional A figure object plotfonts : Dict, optional A dictionary of plot fonts chans : List[str], optional A list of channels to plot label : str, optional Label for the plots xlim : List, optional Limits for the x axis color : str, rgba Tuple The color for the line plot legend : bool Boolean flag for adding a legend Returns ------- plt.figure Matplotlib figure object """ freqArray = self.freqArray fig = (plt.figure(kwargs["fig"].number) if "fig" in kwargs else plt.figure(figsize=(20, 2 * self.numChans))) plotFonts = kwargs[ "plotfonts"] if "plotfonts" in kwargs else getViewFonts() color = kwargs["color"] if "color" in kwargs else None # suptitle st = fig.suptitle( "Spectra data from {} to {}".format(self.startTime, self.stopTime), fontsize=plotFonts["suptitle"], ) st.set_y(0.98) # now plot the data dataChans = kwargs["chans"] if "chans" in kwargs else self.chans numPlotChans = len(dataChans) for idx, chan in enumerate(dataChans): ax = plt.subplot(numPlotChans, 1, idx + 1) plt.title("Channel {}".format(chan), fontsize=plotFonts["title"]) # plot the data if "label" in kwargs: plt.plot( freqArray, np.absolute(self.data[chan]), color=color, label=kwargs["label"], ) else: plt.plot(freqArray, np.absolute(self.data[chan]), color=color) # add frequency label if idx == numPlotChans - 1: plt.xlabel("Frequency [Hz]", fontsize=plotFonts["axisLabel"]) # x axis options xlim = kwargs["xlim"] if "xlim" in kwargs else [ freqArray[0], freqArray[-1] ] plt.xlim(xlim) # y axis options if isElectric(chan): plt.ylabel("[mV/km]", fontsize=plotFonts["axisLabel"]) else: plt.ylabel("[nT]", fontsize=plotFonts["axisLabel"]) plt.grid() # set tick sizes for label in ax.get_xticklabels() + ax.get_yticklabels(): label.set_fontsize(plotFonts["axisTicks"]) # legend if "legend" in kwargs and kwargs["legend"]: plt.legend(loc=4) # show if the figure is not in keywords if "fig" not in kwargs: plt.tight_layout(rect=[0, 0.02, 1, 0.96]) plt.show() return fig
def readHeaderXTR(self, headerFile: str) -> None: """Read a XTR header file The raw data for SPAM is in single precision Volts. However, if there are multiple data files for a single recording, each one may have a different gain. Therefore, a scaling has to be calculated for each data file and channel. This scaling will convert all channels to mV. For the most part, this method only reads recording information. However, it does additionally calculate out the lsb scaling and store it in the ts_lsb channel header. More information is provided in the notes. Notes ----- The raw data for SPAM is in single precision floats and record the raw Voltage measurements of the sensors. However, if there are multiple data files for a single continuous recording, each one may have a different gain. Therefore, a scaling has to be calculated for each data file. For electric channels, the scaling begins with the scaling provided in the header file in the DATA section. This incorporates any gain occuring in the device. This scaling is further amended by a conversion to mV and polarity reversal, .. math:: scaling = read scaling from DATA section of header file \\ scaling = 1000 * scaling , \\ scaling = -1000 * scaling , \\ ts_lsb = scaling , where the reason for the 1000 factor in line 2 is not clear, nor is the polarity reversal. However, this information was provided by people more familiar with the data format. For magnetic channels, the scaling in the header file DATA section is ignored. This is because it includes a static gain correction, which would be duplicated at the calibration stage. Therefore, this is not included at this point. .. math:: scaling = -1000 , \\ ts_lsb = scaling , This scaling converts the magnetic data from V to mV. Parameters ---------- headerFile : str The XTR header file to read in """ with open(headerFile, "r") as f: lines = f.readlines() sectionLines = {} # let's get data for line in lines: line = line.strip() line = line.replace("'", " ") # continue if line is empty if line == "": continue if "[" in line: sec = line[1:-1] sectionLines[sec] = [] else: sectionLines[sec].append(line) # the base class is built around a set of headers based on ATS headers # though this is a bit more work here, it saves lots of code repetition headers = {} # recording information (start_time, start_date, stop_time, stop_date, ats_data_file) fileLine = sectionLines["FILE"][0] fileSplit = fileLine.split() headers["sample_freq"] = np.absolute(float(fileSplit[-1])) timeLine = sectionLines["FILE"][2] timeSplit = timeLine.split() # these are the unix time stamps startDate = float(timeSplit[1] + "." + timeSplit[2]) datetimeStart = datetime.utcfromtimestamp(startDate) stopDate = float(timeSplit[3] + "." + timeSplit[4]) datetimeStop = datetime.utcfromtimestamp(stopDate) headers["start_date"] = datetimeStart.strftime("%Y-%m-%d") headers["start_time"] = datetimeStart.strftime("%H:%M:%S.%f") headers["stop_date"] = datetimeStop.strftime("%Y-%m-%d") headers["stop_time"] = datetimeStop.strftime("%H:%M:%S.%f") # here calculate number of samples deltaSeconds = (datetimeStop - datetimeStart).total_seconds() # calculate number of samples - have to add one because the time given in SPAM recording is the actual time of the last sample numSamples = int(deltaSeconds * headers["sample_freq"]) + 1 # put these in headers for ease of future calculations in merge headers headers["num_samples"] = numSamples # spam datasets only have the one data file for all channels headers["ats_data_file"] = fileSplit[1] # data information (meas_channels, sample_freq) chanLine = sectionLines["CHANNAME"][0] # this gets reformatted to an int later headers["meas_channels"] = chanLine.split()[1] numChansInt = int(headers["meas_channels"]) # deal with the channel headers chanHeaders = [] for iChan in range(0, numChansInt): chanH = self.chanDefaults() # set the sample frequency from the main headers chanH["sample_freq"] = headers["sample_freq"] # line data - read through the data in the correct channel order chanLine = sectionLines["CHANNAME"][iChan + 1] chanSplit = chanLine.split() dataLine = sectionLines["DATA"][iChan + 1] dataSplit = dataLine.split() # channel input information (gain_stage1, gain_stage2, hchopper, echopper) chanH["gain_stage1"] = 1 chanH["gain_stage2"] = 1 # channel output information (sensor_type, channel_type, ts_lsb, pos_x1, pos_x2, pos_y1, pos_y2, pos_z1, pos_z2, sensor_sernum) chanH["ats_data_file"] = fileSplit[1] chanH["num_samples"] = numSamples # channel information # spams often use Bx, By - use H within the software as a whole chanH["channel_type"] = consistentChans(chanSplit[2]) # the sensor number is a bit of a hack - want MFSXXe or something - add MFS in front of the sensor number - this is liable to break # at the same time, set the chopper calLine = sectionLines["200{}003".format(iChan + 1)][0] calSplit = calLine.split() if isMagnetic(chanH["channel_type"]): chanH["sensor_sernum"] = calSplit[ 2] # the last three digits is the serial number sensorType = calSplit[1].split("_")[1][-2:] chanH["sensor_type"] = "MFS{:02d}".format(int(sensorType)) if "LF" in calSplit[1]: chanH["hchopper"] = 1 else: chanH["sensor_type"] = "ELC00" if "LF" in calLine: chanH["echopper"] = 1 # data is raw voltage of sensors # both E and H fields need polarity reversal (from email with Reinhard) # get scaling from headers scaling = float(dataSplit[-2]) if isElectric(chanH["channel_type"]): # the factor of 1000 is not entirely clear lsb = 1000.0 * scaling # volts to millivolts and a minus to switch polarity giving data in mV lsb = -1000.0 * lsb else: # volts to millivolts and a minus to switch polarity giving data in mV # scaling in header file is ignored because it duplicates static gain correction in calibration lsb = -1000.0 chanH["ts_lsb"] = lsb # the distances if chanSplit[2] == "Ex": chanH["pos_x1"] = float(dataSplit[4]) / 2 chanH["pos_x2"] = chanH["pos_x1"] if chanSplit[2] == "Ey": chanH["pos_y1"] = float(dataSplit[4]) / 2 chanH["pos_y2"] = chanH["pos_y1"] if chanSplit[2] == "Ez": chanH["pos_z1"] = float(dataSplit[4]) / 2 chanH["pos_z2"] = chanH["pos_z1"] # append chanHeaders to the list chanHeaders.append(chanH) # check information from raw file headers self.headersFromRawFile(headers["ats_data_file"], headers) # return the headers and chanHeaders from this file return headers, chanHeaders
def getPhysicalSamples(self, **kwargs): """Get data scaled to physical values resistics uses field units, meaning physical samples will return the following: - Electrical channels in mV/km - Magnetic channels in mV - To get magnetic fields in nT, calibration needs to be performed Notes ----- Once Lemi B423 data is scaled (which optionally happens in getUnscaledSamples), the magnetic channels is in mV with gain applied and the electric channels is uV (microvolts). Therefore: - Electric channels need to divided by 1000 along with dipole length division in km (east-west spacing and north-south spacing) to return mV/km. - Magnetic channels need to be divided by the internal gain value which should be set in the headers To get magnetic fields in nT, they have to be calibrated. Parameters ---------- chans : List[str] List of channels to return if not all are required startSample : int First sample to return endSample : int Last sample to return remaverage : bool Remove average from the data remzeros : bool Remove zeroes from the data remnans: bool Remove NanNs from the data Returns ------- TimeData Time data object """ # initialise chans, startSample and endSample with the whole dataset options = self.parseGetDataKeywords(kwargs) # get unscaled data but with gain scalings applied timeData = self.getUnscaledSamples( chans=options["chans"], startSample=options["startSample"], endSample=options["endSample"], scale=True, ) # convert to field units and divide by dipole lengths for chan in options["chans"]: if isElectric(chan): timeData.data[chan] = timeData.data[chan] / 1000.0 timeData.addComment( "Dividing channel {} by 1000 to convert microvolt to millivolt" .format(chan)) if isMagnetic(chan): timeData.data[chan] = timeData.data[chan] / self.getChanGain1( chan) timeData.addComment( "Removing gain of {} from channel {}".format( self.getChanGain1(chan), chan)) if chan == "Ex": # multiply by 1000/self.getChanDx same as dividing by dist in km timeData.data[chan] = (1000.0 * timeData.data[chan] / self.getChanDx(chan)) timeData.addComment( "Dividing channel {} by electrode distance {} km to give mV/km" .format(chan, self.getChanDx(chan) / 1000.0)) if chan == "Ey": # multiply by 1000/self.getChanDy same as dividing by dist in km timeData.data[ chan] = 1000 * timeData.data[chan] / self.getChanDy(chan) timeData.addComment( "Dividing channel {} by electrode distance {:.6f} km to give mV/km" .format(chan, self.getChanDy(chan) / 1000.0)) # if remove zeros - False by default if options["remzeros"]: timeData.data[chan] = removeZerosSingle(timeData.data[chan]) # if remove nans - False by default if options["remnans"]: timeData.data[chan] = removeNansSingle(timeData.data[chan]) # remove the average from the data - True by default if options["remaverage"]: timeData.data[chan] = timeData.data[chan] - np.average( timeData.data[chan]) # add comments timeData.addComment( "Remove zeros: {}, remove nans: {}, remove average: {}".format( options["remzeros"], options["remnans"], options["remaverage"])) return timeData