def projectSpecCalc(proj, **kwargs): generalPrint( "ProjectSpecCalc", "Calculating spectra for project with options: {}".format(kwargs)) # default options options = parseKeywords(getDefaultOptions(proj), kwargs) # calculate the spectra # get the reference time datetimeRef = proj.getRefTime() # prepare the calibrator cal = Calibrator(proj.getCalDataPath()) if options["calibrate"]: cal.printInfo() # loop over sites for s in options["sites"]: # print site info proj.printSiteInfo(s) # get measurement directories for site s timeMeas = proj.getSiteTimeFiles(s) # loop over measurement folders and calculate spectra for each one for meas in timeMeas: # get measurement sample frequency fs = proj.getMeasSampleFreq(s, meas) # check to see if in given frequency list if int(fs) not in options["freqs"]: continue # print measurement info proj.printMeasInfo(s, meas) # get measurement start and end times datetimeStart = proj.getMeasStartTime(s, meas) datetimeEnd = proj.getMeasEndTime(s, meas) # get data, sensor info, serial info, chopper info for calibration reader = proj.getMeasDataReader(s, meas) # get data start and end times - these may not be equal to startDate and endDate # there is the issue of ats data recording end time as one sample late # hence get the actual end time dataStartTime, dataEndTime = reader.getDataTimes( datetimeStart, datetimeEnd) dataChans = reader.getChannels() if len(options["chans"]) > 0: dataChans = options["chans"] # alternatively, could simply do getPhysicalSamples() and get all data that way data = reader.getPhysicalData(dataStartTime, dataEndTime, chans=dataChans) if options["calibrate"]: # do the calibration here sensors = reader.getSensors(dataChans) serials = reader.getSerials(dataChans) choppers = reader.getChoppers(dataChans) data = cal.calibrate(data, fs, sensors, serials, choppers) # notch filter if required for n in options["notch"]: for c in data: data[c] = notchFilter(data[c], fs, n, n / 5.0) # define decimation parameters decParams = DecimationParams(fs) if len(options["evalfreq"]) == 0: decParams.setDecimationParams(options["declevels"], options["freqlevel"]) else: decParams.setFrequencyParams(options["evalfreq"], options["declevels"], options["freqlevel"]) decParams.printInfo() numLevels = decParams.getNumLevels() # now do window parameters winParams = WindowParams(decParams) # winParams.printInfo() # create the decimator dec = Decimator(data, fs, decParams) # dec.printInfo() # loop through decimation levels for iDec in xrange(0, numLevels): # get the data for the current level check = dec.incrementLevel() if not check: break # not enough data #dec.printInfo() data = dec.getData() # create the windower and give it window parameters for current level fsDec = dec.getSampleFreq() win = Windower(datetimeRef, dataStartTime, data, fsDec, winParams.getWindowSize(iDec), winParams.getOverlap(iDec)) numWindows = win.getNumWindows() # win.printInfo() if numWindows < 2: break # do no more decimation # create the spectrum calculator and statistics calculators specCalc = SpectrumCalculator(fsDec, winParams.getWindowSize(iDec)) # get ready a file to save the spectra specWrite = SpectrumWriter(proj.getSpecDataPathMeas(s, meas), datetimeRef) specWrite.openBinaryForWriting("spectra", iDec, fsDec, winParams.getWindowSize(iDec), winParams.getOverlap(iDec), win.getGlobalWindowOffset(), numWindows, dataChans) # loop though windows, calculate spectra and save for iW in xrange(0, numWindows): # get the window data winData = win.getData(iW) # calculate spectra f, specData = specCalc.calcFourierCoeff(winData) # write out spectra specWrite.writeBinary(specData, iW) # close spectra and stat files specWrite.closeFile()
def projectViewTime(proj, startDate, endDate, **kwargs): generalPrint("ProjectViewTime", "Showing time data with options: {}".format(kwargs)) # get options options = parseKeywords(getDefaultOptions(proj), kwargs) # format startDate and endDate start = datetime.strptime("{}.000".format(startDate), "%Y-%m-%d %H:%M:%S.%f") end = datetime.strptime("{}.000".format(endDate), "%Y-%m-%d %H:%M:%S.%f") # create a calibrator in case cal = Calibrator(proj.getCalDataPath()) if options["calibrate"]: cal.printInfo() dataAll = {} xAll = {} fsAll = {} # first need to collect the relevant data # loop over sites for s in options["sites"]: # print site info # proj.printSiteInfo(s) # get measurement directories for site s timeMeas = proj.getSiteTimeFiles(s) # get the dictionary ready dataAll[s] = {} xAll[s] = {} fsAll[s] = {} # loop over measurement folders and calculate spectra for each one for meas in timeMeas: # get measurement sample frequency fs = proj.getMeasSampleFreq(s, meas) # check to see if in given frequency list if int(fs) not in options["freqs"]: continue # now find recordings that are within the recording time siteStart = proj.getMeasStartTime(s, meas) siteEnd = proj.getMeasEndTime(s, meas) if siteEnd < start or siteStart > end: continue # now get the data reader = proj.getMeasDataReader(s, meas) reader.printInfo() # get the samples of the datetimes sampleStart, sampleEnd = reader.time2sample(start, end) # and then go back and get the times - this protects against non second sampling # as the samples returned from time2sample are rounded # using sample2time, we get the appropriate start and end times for those samples readStart, readEnd = reader.sample2time(sampleStart, sampleEnd) # get the data # data = reader.getPhysicalData(readStart, readEnd, chans=options["chans"]) data = reader.getPhysicalData(readStart, readEnd, chans=options["chans"]) # if calibration is on, calibrate the data if options["calibrate"]: generalPrint( "ProjectViewTime", "Calibrating time data: site {}, time data {}".format( s, meas)) sensors = reader.getSensors(reader.getChannels()) serials = reader.getSerials(reader.getChannels()) choppers = reader.getChoppers(reader.getChannels()) data = cal.calibrate(data, fs, sensors, serials, choppers) # have the data, now want to calculate the x array samples = data[options["chans"][0]].size fsDelta = timedelta(seconds=1.0 / fs) x = np.empty(shape=(samples), dtype=datetime) for i in xrange(0, samples): x[i] = readStart + timedelta(seconds=1.0 * i / fs) # save the data dataAll[s][meas] = data xAll[s][meas] = x fsAll[s][meas] = fs # once all the data has been collected, plot it all fig = plt.figure(figsize=options["figsize"]) plotFonts = options["plotfonts"] # suptitle st = fig.suptitle("Time data from {} to {}".format( start.strftime("%Y-%m-%d %H-%M-%S"), end.strftime("%Y-%m-%d %H-%M-%S")), fontsize=plotFonts["suptitle"]) st.set_y(0.98) for s in dataAll: for meas in dataAll[s]: # apply the filter options if options["lpfilt"]: dataAll[s][meas] = lpFilter(dataAll[s][meas], fsAll[s][meas], options["lpfilt"]) if options["hpfilt"]: dataAll[s][meas] = hpFilter(dataAll[s][meas], fsAll[s][meas], options["hpfilt"]) if options["bpfilt"]: dataAll[s][meas] = hpFilter(dataAll[s][meas], fsAll[s][meas], options["bpfilt"][0], options["bpfilt"][1]) generalPrint( "ProjectViewTime", "Plotting {} - {} from {} to {}".format( s, meas, xAll[s][meas][0], xAll[s][meas][-1])) # now plot the data for idx, chan in enumerate(options["chans"]): plt.subplot(4, 1, idx + 1) plotData = dataAll[s][meas][chan] if options["normalise"]: # then normalise the data plotData = plotData / np.linalg.norm(plotData) plt.plot(xAll[s][meas], plotData, label="{} - {}".format(s, meas)) for idx, chan in enumerate(options["chans"]): ax = plt.subplot(4, 1, idx + 1) plt.title("Channel {}".format(chan), fontsize=plotFonts["title"]) plt.grid() if idx == len(options["chans"]) - 1: plt.xlabel("Time", fontsize=plotFonts["axisLabel"]) # limit the x-axis plt.xlim([start, end]) # do the yaxis if isElectric(chan): plt.ylabel("mV/km", fontsize=plotFonts["axisLabel"]) if len(options["Eylim"]) > 0: plt.ylim(options["Eylim"]) else: if options["calibrate"]: plt.ylabel("nT", fontsize=plotFonts["axisLabel"]) else: plt.ylabel("mV", fontsize=plotFonts["axisLabel"]) if len(options["Hylim"]) > 0: plt.ylim(options["Hylim"]) plt.legend(fontsize=plotFonts["legend"]) # set tick sizes for label in (ax.get_xticklabels() + ax.get_yticklabels()): label.set_fontsize(plotFonts["axisTicks"]) plt.tight_layout() # shift subplots down, make room for suptitle fig.subplots_adjust(top=0.92) if options["save"]: fig.savefig( os.path.join( proj.getImageDataPath(), "timeData_{}_{}".format(start.strftime("%Y-%m-%d_%H-%M-%S_"), end.strftime("%Y-%m-%d_%H-%M-%S")))) if options["show"]: plt.show() plt.close("all") return fig
def projectGetTime(proj, site, meas, **kwargs): # print project information proj.printInfo() # print site info proj.printSiteInfo(site) # print measurement info proj.printMeasInfo(site, meas) # get measurement sample frequency fs = proj.getMeasSampleFreq(site, meas) # get measurement start and end times datetimeStart = proj.getMeasStartTime(site, meas) datetimeEnd = proj.getMeasEndTime(site, meas) # get the measurement reader and chans reader = proj.getMeasDataReader(site, meas) chans = reader.getChannels() # set the default parameters decimation = [1] calibrate = False notch = [] detrend = True # apply options if "chans" in kwargs: chans = kwargs["chans"] if "start" in kwargs: datetimeStart = datetime.strptime(kwargs["start"], '%Y-%m-%d %H:%M:%S') if "stop" in kwargs: datetimeEnd = datetime.strptime(kwargs["stop"], '%Y-%m-%d %H:%M:%S') if "decimation" in kwargs: if type(kwargs["decimation"]) != type(list()): decimation = [kwargs["decimation"]] else: decimation = kwargs["decimation"] if "calibrate" in kwargs: calibrate = kwargs["calibrate"] if "detrend" in kwargs: calibrate = kwargs["detrend"] if "notch" in kwargs: notch = kwargs["notch"] # get correct data start and end times dataStartTime, dataEndTime = reader.getDataTimes(datetimeStart, datetimeEnd) # get data, sensor info, serial info, chopper info for calibration data = reader.getPhysicalData(chans, dataStartTime, dataEndTime) sensors = reader.getSensors(chans) serials = reader.getSerials(chans) choppers = reader.getChoppers(chans) if calibrate: cal = Calibrator(proj.getCalDataPath()) cal.printInfo() # calibrate data = cal.calibrate(data, fs, sensors, serials, choppers) # notch filter if required for n in notch: for c in data: data[c] = notchFilter(data[c], fs, n, n/5.0) # lists for saving the data timeData = [] timeX = [] specData = [] specX = [] sampleFreqs = [] # set fsDec fsDec = fs # now for each decimation, get data for d in decimation: # decimate data = downsampleTime(data, d, 51) fsDec = fsDec/d size = data[chans[0]].size # create spectrum calculator specCalc = SpectrumCalculator(fsDec, size) # now calculate the spectrum f, fData = specCalc.calcFourierCoeff(data) # save data if detrend: for c in list(data.keys()): data[c] = signal.detrend(data[c], type="linear") timeData.append(copy.deepcopy(data)) timeX.append([dataStartTime, dataEndTime]) specData.append(fData) specX.append(f) sampleFreqs.append(fsDec) return timeX, timeData, specX, specData
def projectInterpResamp(proj, **kwargs): generalPrint( "Project Interp Resamp", "Resampling / Interpolating project time files with options: {}". format(kwargs)) # resample info is a dictionary # {sampleRateToResample: sampleRateToResampleTo} # this function then does this for all measurement directories of that sample rate in the project options = parseKeywords(getDefaultOptions(proj), kwargs) # create a data calibrator and writer instance cal = Calibrator(proj.getCalDataPath()) if options["calibrate"]: cal.printInfo() writer = DataWriterInternal() # loop over sites for s in options["sites"]: # print site info proj.printSiteInfo(s) for fs in options["freqs"]: timeFiles = proj.getSiteTimeFilesFs(s, fs) if len(timeFiles) == 0: continue # nothing to process # otherwise, resample for tF in timeFiles: # get the reader reader = proj.getMeasDataReader(s, tF) reader.printInfo() # get the dates for the data timeStart = reader.getStartDatetime() timeStop = reader.getStopDatetime() # now check the user provided dates # don't change timeStart, timeEnd yet because that breaks the checking if options["start"]: startUser = datetime.strptime(options["start"], "%Y-%m-%d %H:%M:%S") if startUser > timeStop: # this data has nothing to contribute in the optional date range continue if options["stop"]: stopUser = datetime.strptime(options["stop"], "%Y-%m-%d %H:%M:%S") if stopUser < timeStart: # this data has nothing to contribute in the optional date range continue # if the data contributes, copy in the data if relevant if options["start"]: timeStart = datetime.strptime(options["start"], "%Y-%m-%d %H:%M:%S") if options["stop"]: timeStop = datetime.strptime(options["stop"], "%Y-%m-%d %H:%M:%S") # calculate the samples sampleStart, sampleEnd = reader.time2sample( timeStart, timeStop) dataStartTime, dataStopTime = reader.sample2time( sampleStart, sampleEnd ) # need to do it this way round to protect against fractional sampling # now get the data data = reader.getPhysicalSamples(startSample=sampleStart, endSample=sampleEnd) numSamples = reader.getNumSamples() chans = reader.getChannels() headers = reader.getHeaders() chanHeaders, chanMap = reader.getChanHeaders() dataFs = fs # if calibration is on, calibrate the data if options["calibrate"]: generalPrint( "Project Interp Resamp", "Calibrating time data: site {}, time data {}".format( s, tF)) sensors = reader.getSensors(reader.getChannels()) serials = reader.getSerials(reader.getChannels()) choppers = reader.getChoppers(reader.getChannels()) data = cal.calibrate(data, dataFs, sensors, serials, choppers) # Interpolation to the second - do this first # make sure all the data starts on a full second if options["interp"]: if dataStartTime.microsecond != 0: generalPrint( "Project Interp Resamp", "Interpolating to second: site {}, time data {} at {} Hz" .format(s, tF, dataFs)) # the recording is not on the second - NOTE, this will fail with longer sample periods (i.e. greater than a second) startTimeInterp, numSamplesInterp, dataInterp = interpolateToSecond( dataFs, dataStartTime, data) numSamples = numSamplesInterp dataStartTime = startTimeInterp data = dataInterp # Check if fs belongs to options["resamp"] keys and resample if does # resampling does not change the start time - hence dataStartTime is unchanged if dataFs in options[ "resamp"]: # then need to resample this data generalPrint( "Project Interp Resamp", "Resampling site = {}, time data {} at {} Hz to {} Hz". format(s, tF, fs, options["resamp"][fs])) data = resample(data, dataFs, options["resamp"][dataFs]) # update info for saving file dataFs = options["resamp"][fs] numSamples = data[chans[0]].size # recall, start times stay the same with resampling - only the sample rate changes and the number of samples # write out data - the data writer automatically deals with the end date outPath = os.path.join( proj.getTimeDataPathSite(s), options["prepend"] + tF + options["postpend"]) writer.setOutPath(outPath) writer.writeData( headers, chanHeaders, data, start_time=dataStartTime.strftime("%H:%M:%S.%f"), start_date=dataStartTime.strftime("%Y-%m-%d"), numSamples=numSamples, sample_freq=dataFs, lsb_applied=True) writer.printInfo()