def projectSpecCalc(proj, **kwargs):
generalPrint(
"ProjectSpecCalc",
"Calculating spectra for project with options: {}".format(kwargs))
# default options
options = parseKeywords(getDefaultOptions(proj), kwargs)
# calculate the spectra
# get the reference time
datetimeRef = proj.getRefTime()
# prepare the calibrator
cal = Calibrator(proj.getCalDataPath())
if options["calibrate"]:
cal.printInfo()
# loop over sites
for s in options["sites"]:
# print site info
proj.printSiteInfo(s)
# get measurement directories for site s
timeMeas = proj.getSiteTimeFiles(s)
# loop over measurement folders and calculate spectra for each one
for meas in timeMeas:
# get measurement sample frequency
fs = proj.getMeasSampleFreq(s, meas)
# check to see if in given frequency list
if int(fs) not in options["freqs"]:
continue
# print measurement info
proj.printMeasInfo(s, meas)
# get measurement start and end times
datetimeStart = proj.getMeasStartTime(s, meas)
datetimeEnd = proj.getMeasEndTime(s, meas)
# get data, sensor info, serial info, chopper info for calibration
reader = proj.getMeasDataReader(s, meas)
# get data start and end times - these may not be equal to startDate and endDate
# there is the issue of ats data recording end time as one sample late
# hence get the actual end time
dataStartTime, dataEndTime = reader.getDataTimes(
datetimeStart, datetimeEnd)
dataChans = reader.getChannels()
if len(options["chans"]) > 0:
dataChans = options["chans"]
# alternatively, could simply do getPhysicalSamples() and get all data that way
data = reader.getPhysicalData(dataStartTime,
dataEndTime,
chans=dataChans)
if options["calibrate"]:
# do the calibration here
sensors = reader.getSensors(dataChans)
serials = reader.getSerials(dataChans)
choppers = reader.getChoppers(dataChans)
data = cal.calibrate(data, fs, sensors, serials, choppers)
# notch filter if required
for n in options["notch"]:
for c in data:
data[c] = notchFilter(data[c], fs, n, n / 5.0)
# define decimation parameters
decParams = DecimationParams(fs)
if len(options["evalfreq"]) == 0:
decParams.setDecimationParams(options["declevels"],
options["freqlevel"])
else:
decParams.setFrequencyParams(options["evalfreq"],
options["declevels"],
options["freqlevel"])
decParams.printInfo()
numLevels = decParams.getNumLevels()
# now do window parameters
winParams = WindowParams(decParams)
# winParams.printInfo()
# create the decimator
dec = Decimator(data, fs, decParams)
# dec.printInfo()
# loop through decimation levels
for iDec in xrange(0, numLevels):
# get the data for the current level
check = dec.incrementLevel()
if not check:
break # not enough data
#dec.printInfo()
data = dec.getData()
# create the windower and give it window parameters for current level
fsDec = dec.getSampleFreq()
win = Windower(datetimeRef, dataStartTime, data, fsDec,
winParams.getWindowSize(iDec),
winParams.getOverlap(iDec))
numWindows = win.getNumWindows()
# win.printInfo()
if numWindows < 2:
break # do no more decimation
# create the spectrum calculator and statistics calculators
specCalc = SpectrumCalculator(fsDec,
winParams.getWindowSize(iDec))
# get ready a file to save the spectra
specWrite = SpectrumWriter(proj.getSpecDataPathMeas(s, meas),
datetimeRef)
specWrite.openBinaryForWriting("spectra", iDec, fsDec,
winParams.getWindowSize(iDec),
winParams.getOverlap(iDec),
win.getGlobalWindowOffset(),
numWindows, dataChans)
# loop though windows, calculate spectra and save
for iW in xrange(0, numWindows):
# get the window data
winData = win.getData(iW)
# calculate spectra
f, specData = specCalc.calcFourierCoeff(winData)
# write out spectra
specWrite.writeBinary(specData, iW)
# close spectra and stat files
specWrite.closeFile()
def projectViewTime(proj, startDate, endDate, **kwargs):
generalPrint("ProjectViewTime",
"Showing time data with options: {}".format(kwargs))
# get options
options = parseKeywords(getDefaultOptions(proj), kwargs)
# format startDate and endDate
start = datetime.strptime("{}.000".format(startDate),
"%Y-%m-%d %H:%M:%S.%f")
end = datetime.strptime("{}.000".format(endDate), "%Y-%m-%d %H:%M:%S.%f")
# create a calibrator in case
cal = Calibrator(proj.getCalDataPath())
if options["calibrate"]:
cal.printInfo()
dataAll = {}
xAll = {}
fsAll = {}
# first need to collect the relevant data
# loop over sites
for s in options["sites"]:
# print site info
# proj.printSiteInfo(s)
# get measurement directories for site s
timeMeas = proj.getSiteTimeFiles(s)
# get the dictionary ready
dataAll[s] = {}
xAll[s] = {}
fsAll[s] = {}
# loop over measurement folders and calculate spectra for each one
for meas in timeMeas:
# get measurement sample frequency
fs = proj.getMeasSampleFreq(s, meas)
# check to see if in given frequency list
if int(fs) not in options["freqs"]:
continue
# now find recordings that are within the recording time
siteStart = proj.getMeasStartTime(s, meas)
siteEnd = proj.getMeasEndTime(s, meas)
if siteEnd < start or siteStart > end:
continue
# now get the data
reader = proj.getMeasDataReader(s, meas)
reader.printInfo()
# get the samples of the datetimes
sampleStart, sampleEnd = reader.time2sample(start, end)
# and then go back and get the times - this protects against non second sampling
# as the samples returned from time2sample are rounded
# using sample2time, we get the appropriate start and end times for those samples
readStart, readEnd = reader.sample2time(sampleStart, sampleEnd)
# get the data
# data = reader.getPhysicalData(readStart, readEnd, chans=options["chans"])
data = reader.getPhysicalData(readStart,
readEnd,
chans=options["chans"])
# if calibration is on, calibrate the data
if options["calibrate"]:
generalPrint(
"ProjectViewTime",
"Calibrating time data: site {}, time data {}".format(
s, meas))
sensors = reader.getSensors(reader.getChannels())
serials = reader.getSerials(reader.getChannels())
choppers = reader.getChoppers(reader.getChannels())
data = cal.calibrate(data, fs, sensors, serials, choppers)
# have the data, now want to calculate the x array
samples = data[options["chans"][0]].size
fsDelta = timedelta(seconds=1.0 / fs)
x = np.empty(shape=(samples), dtype=datetime)
for i in xrange(0, samples):
x[i] = readStart + timedelta(seconds=1.0 * i / fs)
# save the data
dataAll[s][meas] = data
xAll[s][meas] = x
fsAll[s][meas] = fs
# once all the data has been collected, plot it all
fig = plt.figure(figsize=options["figsize"])
plotFonts = options["plotfonts"]
# suptitle
st = fig.suptitle("Time data from {} to {}".format(
start.strftime("%Y-%m-%d %H-%M-%S"),
end.strftime("%Y-%m-%d %H-%M-%S")),
fontsize=plotFonts["suptitle"])
st.set_y(0.98)
for s in dataAll:
for meas in dataAll[s]:
# apply the filter options
if options["lpfilt"]:
dataAll[s][meas] = lpFilter(dataAll[s][meas], fsAll[s][meas],
options["lpfilt"])
if options["hpfilt"]:
dataAll[s][meas] = hpFilter(dataAll[s][meas], fsAll[s][meas],
options["hpfilt"])
if options["bpfilt"]:
dataAll[s][meas] = hpFilter(dataAll[s][meas], fsAll[s][meas],
options["bpfilt"][0],
options["bpfilt"][1])
generalPrint(
"ProjectViewTime", "Plotting {} - {} from {} to {}".format(
s, meas, xAll[s][meas][0], xAll[s][meas][-1]))
# now plot the data
for idx, chan in enumerate(options["chans"]):
plt.subplot(4, 1, idx + 1)
plotData = dataAll[s][meas][chan]
if options["normalise"]: # then normalise the data
plotData = plotData / np.linalg.norm(plotData)
plt.plot(xAll[s][meas],
plotData,
label="{} - {}".format(s, meas))
for idx, chan in enumerate(options["chans"]):
ax = plt.subplot(4, 1, idx + 1)
plt.title("Channel {}".format(chan), fontsize=plotFonts["title"])
plt.grid()
if idx == len(options["chans"]) - 1:
plt.xlabel("Time", fontsize=plotFonts["axisLabel"])
# limit the x-axis
plt.xlim([start, end])
# do the yaxis
if isElectric(chan):
plt.ylabel("mV/km", fontsize=plotFonts["axisLabel"])
if len(options["Eylim"]) > 0:
plt.ylim(options["Eylim"])
else:
if options["calibrate"]:
plt.ylabel("nT", fontsize=plotFonts["axisLabel"])
else:
plt.ylabel("mV", fontsize=plotFonts["axisLabel"])
if len(options["Hylim"]) > 0:
plt.ylim(options["Hylim"])
plt.legend(fontsize=plotFonts["legend"])
# set tick sizes
for label in (ax.get_xticklabels() + ax.get_yticklabels()):
label.set_fontsize(plotFonts["axisTicks"])
plt.tight_layout()
# shift subplots down, make room for suptitle
fig.subplots_adjust(top=0.92)
if options["save"]:
fig.savefig(
os.path.join(
proj.getImageDataPath(),
"timeData_{}_{}".format(start.strftime("%Y-%m-%d_%H-%M-%S_"),
end.strftime("%Y-%m-%d_%H-%M-%S"))))
if options["show"]:
plt.show()
plt.close("all")
return fig
def projectGetTime(proj, site, meas, **kwargs): # print project information proj.printInfo() # print site info proj.printSiteInfo(site) # print measurement info proj.printMeasInfo(site, meas) # get measurement sample frequency fs = proj.getMeasSampleFreq(site, meas) # get measurement start and end times datetimeStart = proj.getMeasStartTime(site, meas) datetimeEnd = proj.getMeasEndTime(site, meas) # get the measurement reader and chans reader = proj.getMeasDataReader(site, meas) chans = reader.getChannels() # set the default parameters decimation = [1] calibrate = False notch = [] detrend = True # apply options if "chans" in kwargs: chans = kwargs["chans"] if "start" in kwargs: datetimeStart = datetime.strptime(kwargs["start"], '%Y-%m-%d %H:%M:%S') if "stop" in kwargs: datetimeEnd = datetime.strptime(kwargs["stop"], '%Y-%m-%d %H:%M:%S') if "decimation" in kwargs: if type(kwargs["decimation"]) != type(list()): decimation = [kwargs["decimation"]] else: decimation = kwargs["decimation"] if "calibrate" in kwargs: calibrate = kwargs["calibrate"] if "detrend" in kwargs: calibrate = kwargs["detrend"] if "notch" in kwargs: notch = kwargs["notch"] # get correct data start and end times dataStartTime, dataEndTime = reader.getDataTimes(datetimeStart, datetimeEnd) # get data, sensor info, serial info, chopper info for calibration data = reader.getPhysicalData(chans, dataStartTime, dataEndTime) sensors = reader.getSensors(chans) serials = reader.getSerials(chans) choppers = reader.getChoppers(chans) if calibrate: cal = Calibrator(proj.getCalDataPath()) cal.printInfo() # calibrate data = cal.calibrate(data, fs, sensors, serials, choppers) # notch filter if required for n in notch: for c in data: data[c] = notchFilter(data[c], fs, n, n/5.0) # lists for saving the data timeData = [] timeX = [] specData = [] specX = [] sampleFreqs = [] # set fsDec fsDec = fs # now for each decimation, get data for d in decimation: # decimate data = downsampleTime(data, d, 51) fsDec = fsDec/d size = data[chans[0]].size # create spectrum calculator specCalc = SpectrumCalculator(fsDec, size) # now calculate the spectrum f, fData = specCalc.calcFourierCoeff(data) # save data if detrend: for c in list(data.keys()): data[c] = signal.detrend(data[c], type="linear") timeData.append(copy.deepcopy(data)) timeX.append([dataStartTime, dataEndTime]) specData.append(fData) specX.append(f) sampleFreqs.append(fsDec) return timeX, timeData, specX, specData
def projectInterpResamp(proj, **kwargs):
generalPrint(
"Project Interp Resamp",
"Resampling / Interpolating project time files with options: {}".
format(kwargs))
# resample info is a dictionary
# {sampleRateToResample: sampleRateToResampleTo}
# this function then does this for all measurement directories of that sample rate in the project
options = parseKeywords(getDefaultOptions(proj), kwargs)
# create a data calibrator and writer instance
cal = Calibrator(proj.getCalDataPath())
if options["calibrate"]:
cal.printInfo()
writer = DataWriterInternal()
# loop over sites
for s in options["sites"]:
# print site info
proj.printSiteInfo(s)
for fs in options["freqs"]:
timeFiles = proj.getSiteTimeFilesFs(s, fs)
if len(timeFiles) == 0:
continue # nothing to process
# otherwise, resample
for tF in timeFiles:
# get the reader
reader = proj.getMeasDataReader(s, tF)
reader.printInfo()
# get the dates for the data
timeStart = reader.getStartDatetime()
timeStop = reader.getStopDatetime()
# now check the user provided dates
# don't change timeStart, timeEnd yet because that breaks the checking
if options["start"]:
startUser = datetime.strptime(options["start"],
"%Y-%m-%d %H:%M:%S")
if startUser > timeStop: # this data has nothing to contribute in the optional date range
continue
if options["stop"]:
stopUser = datetime.strptime(options["stop"],
"%Y-%m-%d %H:%M:%S")
if stopUser < timeStart: # this data has nothing to contribute in the optional date range
continue
# if the data contributes, copy in the data if relevant
if options["start"]:
timeStart = datetime.strptime(options["start"],
"%Y-%m-%d %H:%M:%S")
if options["stop"]:
timeStop = datetime.strptime(options["stop"],
"%Y-%m-%d %H:%M:%S")
# calculate the samples
sampleStart, sampleEnd = reader.time2sample(
timeStart, timeStop)
dataStartTime, dataStopTime = reader.sample2time(
sampleStart, sampleEnd
) # need to do it this way round to protect against fractional sampling
# now get the data
data = reader.getPhysicalSamples(startSample=sampleStart,
endSample=sampleEnd)
numSamples = reader.getNumSamples()
chans = reader.getChannels()
headers = reader.getHeaders()
chanHeaders, chanMap = reader.getChanHeaders()
dataFs = fs
# if calibration is on, calibrate the data
if options["calibrate"]:
generalPrint(
"Project Interp Resamp",
"Calibrating time data: site {}, time data {}".format(
s, tF))
sensors = reader.getSensors(reader.getChannels())
serials = reader.getSerials(reader.getChannels())
choppers = reader.getChoppers(reader.getChannels())
data = cal.calibrate(data, dataFs, sensors, serials,
choppers)
# Interpolation to the second - do this first
# make sure all the data starts on a full second
if options["interp"]:
if dataStartTime.microsecond != 0:
generalPrint(
"Project Interp Resamp",
"Interpolating to second: site {}, time data {} at {} Hz"
.format(s, tF, dataFs))
# the recording is not on the second - NOTE, this will fail with longer sample periods (i.e. greater than a second)
startTimeInterp, numSamplesInterp, dataInterp = interpolateToSecond(
dataFs, dataStartTime, data)
numSamples = numSamplesInterp
dataStartTime = startTimeInterp
data = dataInterp
# Check if fs belongs to options["resamp"] keys and resample if does
# resampling does not change the start time - hence dataStartTime is unchanged
if dataFs in options[
"resamp"]: # then need to resample this data
generalPrint(
"Project Interp Resamp",
"Resampling site = {}, time data {} at {} Hz to {} Hz".
format(s, tF, fs, options["resamp"][fs]))
data = resample(data, dataFs, options["resamp"][dataFs])
# update info for saving file
dataFs = options["resamp"][fs]
numSamples = data[chans[0]].size
# recall, start times stay the same with resampling - only the sample rate changes and the number of samples
# write out data - the data writer automatically deals with the end date
outPath = os.path.join(
proj.getTimeDataPathSite(s),
options["prepend"] + tF + options["postpend"])
writer.setOutPath(outPath)
writer.writeData(
headers,
chanHeaders,
data,
start_time=dataStartTime.strftime("%H:%M:%S.%f"),
start_date=dataStartTime.strftime("%Y-%m-%d"),
numSamples=numSamples,
sample_freq=dataFs,
lsb_applied=True)
writer.printInfo()