Ejemplo n.º 1
0
def projectSpecCalc(proj, **kwargs):
    generalPrint(
        "ProjectSpecCalc",
        "Calculating spectra for project with options: {}".format(kwargs))
    # default options
    options = parseKeywords(getDefaultOptions(proj), kwargs)

    # calculate the spectra
    # get the reference time
    datetimeRef = proj.getRefTime()
    # prepare the calibrator
    cal = Calibrator(proj.getCalDataPath())
    if options["calibrate"]:
        cal.printInfo()

    # loop over sites
    for s in options["sites"]:
        # print site info
        proj.printSiteInfo(s)

        # get measurement directories for site s
        timeMeas = proj.getSiteTimeFiles(s)

        # loop over measurement folders and calculate spectra for each one
        for meas in timeMeas:
            # get measurement sample frequency
            fs = proj.getMeasSampleFreq(s, meas)
            # check to see if in given frequency list
            if int(fs) not in options["freqs"]:
                continue

            # print measurement info
            proj.printMeasInfo(s, meas)

            # get measurement start and end times
            datetimeStart = proj.getMeasStartTime(s, meas)
            datetimeEnd = proj.getMeasEndTime(s, meas)

            # get data, sensor info, serial info, chopper info for calibration
            reader = proj.getMeasDataReader(s, meas)
            # get data start and end times - these may not be equal to startDate and endDate
            # there is the issue of ats data recording end time as one sample late
            # hence get the actual end time
            dataStartTime, dataEndTime = reader.getDataTimes(
                datetimeStart, datetimeEnd)
            dataChans = reader.getChannels()
            if len(options["chans"]) > 0:
                dataChans = options["chans"]
            # alternatively, could simply do getPhysicalSamples() and get all data that way
            data = reader.getPhysicalData(dataStartTime,
                                          dataEndTime,
                                          chans=dataChans)

            if options["calibrate"]:
                # do the calibration here
                sensors = reader.getSensors(dataChans)
                serials = reader.getSerials(dataChans)
                choppers = reader.getChoppers(dataChans)
                data = cal.calibrate(data, fs, sensors, serials, choppers)

            # notch filter if required
            for n in options["notch"]:
                for c in data:
                    data[c] = notchFilter(data[c], fs, n, n / 5.0)

            # define decimation parameters
            decParams = DecimationParams(fs)
            if len(options["evalfreq"]) == 0:
                decParams.setDecimationParams(options["declevels"],
                                              options["freqlevel"])
            else:
                decParams.setFrequencyParams(options["evalfreq"],
                                             options["declevels"],
                                             options["freqlevel"])
            decParams.printInfo()
            numLevels = decParams.getNumLevels()

            # now do window parameters
            winParams = WindowParams(decParams)
            # winParams.printInfo()

            # create the decimator
            dec = Decimator(data, fs, decParams)
            # dec.printInfo()

            # loop through decimation levels
            for iDec in xrange(0, numLevels):
                # get the data for the current level
                check = dec.incrementLevel()
                if not check:
                    break  # not enough data
                #dec.printInfo()
                data = dec.getData()

                # create the windower and give it window parameters for current level
                fsDec = dec.getSampleFreq()
                win = Windower(datetimeRef, dataStartTime, data, fsDec,
                               winParams.getWindowSize(iDec),
                               winParams.getOverlap(iDec))
                numWindows = win.getNumWindows()
                # win.printInfo()
                if numWindows < 2:
                    break  # do no more decimation

                # create the spectrum calculator and statistics calculators
                specCalc = SpectrumCalculator(fsDec,
                                              winParams.getWindowSize(iDec))

                # get ready a file to save the spectra
                specWrite = SpectrumWriter(proj.getSpecDataPathMeas(s, meas),
                                           datetimeRef)
                specWrite.openBinaryForWriting("spectra", iDec, fsDec,
                                               winParams.getWindowSize(iDec),
                                               winParams.getOverlap(iDec),
                                               win.getGlobalWindowOffset(),
                                               numWindows, dataChans)

                # loop though windows, calculate spectra and save
                for iW in xrange(0, numWindows):
                    # get the window data
                    winData = win.getData(iW)

                    # calculate spectra
                    f, specData = specCalc.calcFourierCoeff(winData)

                    # write out spectra
                    specWrite.writeBinary(specData, iW)

                # close spectra and stat files
                specWrite.closeFile()
Ejemplo n.º 2
0
def projectViewTime(proj, startDate, endDate, **kwargs):
    generalPrint("ProjectViewTime",
                 "Showing time data with options: {}".format(kwargs))
    # get options
    options = parseKeywords(getDefaultOptions(proj), kwargs)
    # format startDate and endDate
    start = datetime.strptime("{}.000".format(startDate),
                              "%Y-%m-%d %H:%M:%S.%f")
    end = datetime.strptime("{}.000".format(endDate), "%Y-%m-%d %H:%M:%S.%f")

    # create a calibrator in case
    cal = Calibrator(proj.getCalDataPath())
    if options["calibrate"]:
        cal.printInfo()

    dataAll = {}
    xAll = {}
    fsAll = {}
    # first need to collect the relevant data
    # loop over sites
    for s in options["sites"]:
        # print site info
        # proj.printSiteInfo(s)

        # get measurement directories for site s
        timeMeas = proj.getSiteTimeFiles(s)

        # get the dictionary ready
        dataAll[s] = {}
        xAll[s] = {}
        fsAll[s] = {}
        # loop over measurement folders and calculate spectra for each one
        for meas in timeMeas:
            # get measurement sample frequency
            fs = proj.getMeasSampleFreq(s, meas)
            # check to see if in given frequency list
            if int(fs) not in options["freqs"]:
                continue

            # now find recordings that are within the recording time
            siteStart = proj.getMeasStartTime(s, meas)
            siteEnd = proj.getMeasEndTime(s, meas)
            if siteEnd < start or siteStart > end:
                continue

            # now get the data
            reader = proj.getMeasDataReader(s, meas)
            reader.printInfo()
            # get the samples of the datetimes
            sampleStart, sampleEnd = reader.time2sample(start, end)
            # and then go back and get the times - this protects against non second sampling
            # as the samples returned from time2sample are rounded
            # using sample2time, we get the appropriate start and end times for those samples
            readStart, readEnd = reader.sample2time(sampleStart, sampleEnd)
            # get the data
            # data = reader.getPhysicalData(readStart, readEnd, chans=options["chans"])
            data = reader.getPhysicalData(readStart,
                                          readEnd,
                                          chans=options["chans"])
            # if calibration is on, calibrate the data
            if options["calibrate"]:
                generalPrint(
                    "ProjectViewTime",
                    "Calibrating time data: site {}, time data {}".format(
                        s, meas))
                sensors = reader.getSensors(reader.getChannels())
                serials = reader.getSerials(reader.getChannels())
                choppers = reader.getChoppers(reader.getChannels())
                data = cal.calibrate(data, fs, sensors, serials, choppers)

            # have the data, now want to calculate the x array
            samples = data[options["chans"][0]].size
            fsDelta = timedelta(seconds=1.0 / fs)
            x = np.empty(shape=(samples), dtype=datetime)
            for i in xrange(0, samples):
                x[i] = readStart + timedelta(seconds=1.0 * i / fs)
            # save the data
            dataAll[s][meas] = data
            xAll[s][meas] = x
            fsAll[s][meas] = fs

    # once all the data has been collected, plot it all
    fig = plt.figure(figsize=options["figsize"])
    plotFonts = options["plotfonts"]
    # suptitle
    st = fig.suptitle("Time data from {} to {}".format(
        start.strftime("%Y-%m-%d %H-%M-%S"),
        end.strftime("%Y-%m-%d %H-%M-%S")),
                      fontsize=plotFonts["suptitle"])
    st.set_y(0.98)
    for s in dataAll:
        for meas in dataAll[s]:
            # apply the filter options
            if options["lpfilt"]:
                dataAll[s][meas] = lpFilter(dataAll[s][meas], fsAll[s][meas],
                                            options["lpfilt"])
            if options["hpfilt"]:
                dataAll[s][meas] = hpFilter(dataAll[s][meas], fsAll[s][meas],
                                            options["hpfilt"])
            if options["bpfilt"]:
                dataAll[s][meas] = hpFilter(dataAll[s][meas], fsAll[s][meas],
                                            options["bpfilt"][0],
                                            options["bpfilt"][1])

            generalPrint(
                "ProjectViewTime", "Plotting {} - {} from {}  to {}".format(
                    s, meas, xAll[s][meas][0], xAll[s][meas][-1]))
            # now plot the data
            for idx, chan in enumerate(options["chans"]):
                plt.subplot(4, 1, idx + 1)
                plotData = dataAll[s][meas][chan]
                if options["normalise"]:  # then normalise the data
                    plotData = plotData / np.linalg.norm(plotData)
                plt.plot(xAll[s][meas],
                         plotData,
                         label="{} - {}".format(s, meas))

    for idx, chan in enumerate(options["chans"]):
        ax = plt.subplot(4, 1, idx + 1)
        plt.title("Channel {}".format(chan), fontsize=plotFonts["title"])
        plt.grid()
        if idx == len(options["chans"]) - 1:
            plt.xlabel("Time", fontsize=plotFonts["axisLabel"])
        # limit the x-axis
        plt.xlim([start, end])
        # do the yaxis
        if isElectric(chan):
            plt.ylabel("mV/km", fontsize=plotFonts["axisLabel"])
            if len(options["Eylim"]) > 0:
                plt.ylim(options["Eylim"])
        else:
            if options["calibrate"]:
                plt.ylabel("nT", fontsize=plotFonts["axisLabel"])
            else:
                plt.ylabel("mV", fontsize=plotFonts["axisLabel"])
            if len(options["Hylim"]) > 0:
                plt.ylim(options["Hylim"])
        plt.legend(fontsize=plotFonts["legend"])
        # set tick sizes
        for label in (ax.get_xticklabels() + ax.get_yticklabels()):
            label.set_fontsize(plotFonts["axisTicks"])

    plt.tight_layout()
    # shift subplots down, make room for suptitle
    fig.subplots_adjust(top=0.92)
    if options["save"]:
        fig.savefig(
            os.path.join(
                proj.getImageDataPath(),
                "timeData_{}_{}".format(start.strftime("%Y-%m-%d_%H-%M-%S_"),
                                        end.strftime("%Y-%m-%d_%H-%M-%S"))))
    if options["show"]:
        plt.show()
    plt.close("all")
    return fig
Ejemplo n.º 3
0
def projectGetTime(proj, site, meas, **kwargs):
	# print project information
	proj.printInfo()

	# print site info
	proj.printSiteInfo(site)

	# print measurement info
	proj.printMeasInfo(site, meas)

	# get measurement sample frequency
	fs = proj.getMeasSampleFreq(site, meas)
	# get measurement start and end times
	datetimeStart = proj.getMeasStartTime(site, meas)
	datetimeEnd = proj.getMeasEndTime(site, meas)
	# get the measurement reader and chans
	reader = proj.getMeasDataReader(site, meas)
	chans = reader.getChannels()

	# set the default parameters
	decimation = [1]
	calibrate = False
	notch = []
	detrend = True

	# apply options
	if "chans" in kwargs:
		chans = kwargs["chans"]
	if "start" in kwargs:
		datetimeStart = datetime.strptime(kwargs["start"], '%Y-%m-%d %H:%M:%S')
	if "stop" in kwargs:
		datetimeEnd = datetime.strptime(kwargs["stop"], '%Y-%m-%d %H:%M:%S')
	if "decimation" in kwargs:
		if type(kwargs["decimation"]) != type(list()):
			decimation = [kwargs["decimation"]]
		else:
			decimation = kwargs["decimation"]
	if "calibrate" in kwargs:
		calibrate = kwargs["calibrate"]	
	if "detrend" in kwargs:
		calibrate = kwargs["detrend"]			
	if "notch" in kwargs:
		notch = kwargs["notch"]			

	# get correct data start and end times
	dataStartTime, dataEndTime = reader.getDataTimes(datetimeStart, datetimeEnd)

	# get data, sensor info, serial info, chopper info for calibration
	data = reader.getPhysicalData(chans, dataStartTime, dataEndTime)
	sensors = reader.getSensors(chans)
	serials = reader.getSerials(chans)
	choppers = reader.getChoppers(chans)

	if calibrate:
		cal = Calibrator(proj.getCalDataPath())
		cal.printInfo()	
		# calibrate		
		data = cal.calibrate(data, fs, sensors, serials, choppers)

	# notch filter if required
	for n in notch:
		for c in data:
			data[c] = notchFilter(data[c], fs, n, n/5.0)		

	# lists for saving the data
	timeData = []
	timeX = []
	specData = []
	specX = []
	sampleFreqs = []

	# set fsDec
	fsDec = fs
	# now for each decimation, get data
	for d in decimation:
		# decimate
		data = downsampleTime(data, d, 51)
		fsDec = fsDec/d
		size = data[chans[0]].size
		# create spectrum calculator
		specCalc = SpectrumCalculator(fsDec, size)			

		# now calculate the spectrum
		f, fData = specCalc.calcFourierCoeff(data)

		# save data
		if detrend:
			for c in list(data.keys()):
				data[c] = signal.detrend(data[c], type="linear")	
		timeData.append(copy.deepcopy(data))
		timeX.append([dataStartTime, dataEndTime])
		specData.append(fData)
		specX.append(f)
		sampleFreqs.append(fsDec)	

	return timeX, timeData, specX, specData
Ejemplo n.º 4
0
def projectInterpResamp(proj, **kwargs):
    generalPrint(
        "Project Interp Resamp",
        "Resampling / Interpolating project time files with options: {}".
        format(kwargs))
    # resample info is a dictionary
    # {sampleRateToResample: sampleRateToResampleTo}
    # this function then does this for all measurement directories of that sample rate in the project
    options = parseKeywords(getDefaultOptions(proj), kwargs)

    # create a data calibrator and writer instance
    cal = Calibrator(proj.getCalDataPath())
    if options["calibrate"]:
        cal.printInfo()
    writer = DataWriterInternal()

    # loop over sites
    for s in options["sites"]:
        # print site info
        proj.printSiteInfo(s)

        for fs in options["freqs"]:
            timeFiles = proj.getSiteTimeFilesFs(s, fs)

            if len(timeFiles) == 0:
                continue  # nothing to process

            # otherwise, resample
            for tF in timeFiles:
                # get the reader
                reader = proj.getMeasDataReader(s, tF)
                reader.printInfo()
                # get the dates for the data
                timeStart = reader.getStartDatetime()
                timeStop = reader.getStopDatetime()

                # now check the user provided dates
                # don't change timeStart, timeEnd yet because that breaks the checking
                if options["start"]:
                    startUser = datetime.strptime(options["start"],
                                                  "%Y-%m-%d %H:%M:%S")
                    if startUser > timeStop:  # this data has nothing to contribute in the optional date range
                        continue
                if options["stop"]:
                    stopUser = datetime.strptime(options["stop"],
                                                 "%Y-%m-%d %H:%M:%S")
                    if stopUser < timeStart:  # this data has nothing to contribute in the optional date range
                        continue

                # if the data contributes, copy in the data if relevant
                if options["start"]:
                    timeStart = datetime.strptime(options["start"],
                                                  "%Y-%m-%d %H:%M:%S")
                if options["stop"]:
                    timeStop = datetime.strptime(options["stop"],
                                                 "%Y-%m-%d %H:%M:%S")

                # calculate the samples
                sampleStart, sampleEnd = reader.time2sample(
                    timeStart, timeStop)
                dataStartTime, dataStopTime = reader.sample2time(
                    sampleStart, sampleEnd
                )  # need to do it this way round to protect against fractional sampling
                # now get the data
                data = reader.getPhysicalSamples(startSample=sampleStart,
                                                 endSample=sampleEnd)
                numSamples = reader.getNumSamples()
                chans = reader.getChannels()
                headers = reader.getHeaders()
                chanHeaders, chanMap = reader.getChanHeaders()
                dataFs = fs

                # if calibration is on, calibrate the data
                if options["calibrate"]:
                    generalPrint(
                        "Project Interp Resamp",
                        "Calibrating time data: site {}, time data {}".format(
                            s, tF))
                    sensors = reader.getSensors(reader.getChannels())
                    serials = reader.getSerials(reader.getChannels())
                    choppers = reader.getChoppers(reader.getChannels())
                    data = cal.calibrate(data, dataFs, sensors, serials,
                                         choppers)

                # Interpolation to the second - do this first
                # make sure all the data starts on a full second
                if options["interp"]:
                    if dataStartTime.microsecond != 0:
                        generalPrint(
                            "Project Interp Resamp",
                            "Interpolating to second: site {}, time data {} at {} Hz"
                            .format(s, tF, dataFs))
                        # the recording is not on the second - NOTE, this will fail with longer sample periods (i.e. greater than a second)
                        startTimeInterp, numSamplesInterp, dataInterp = interpolateToSecond(
                            dataFs, dataStartTime, data)
                        numSamples = numSamplesInterp
                        dataStartTime = startTimeInterp
                        data = dataInterp

                # Check if fs belongs to options["resamp"] keys and resample if does
                # resampling does not change the start time - hence dataStartTime is unchanged
                if dataFs in options[
                        "resamp"]:  # then need to resample this data
                    generalPrint(
                        "Project Interp Resamp",
                        "Resampling site = {}, time data {} at {} Hz to {} Hz".
                        format(s, tF, fs, options["resamp"][fs]))
                    data = resample(data, dataFs, options["resamp"][dataFs])
                    # update info for saving file
                    dataFs = options["resamp"][fs]
                    numSamples = data[chans[0]].size

                # recall, start times stay the same with resampling - only the sample rate changes and the number of samples
                # write out data - the data writer automatically deals with the end date
                outPath = os.path.join(
                    proj.getTimeDataPathSite(s),
                    options["prepend"] + tF + options["postpend"])
                writer.setOutPath(outPath)
                writer.writeData(
                    headers,
                    chanHeaders,
                    data,
                    start_time=dataStartTime.strftime("%H:%M:%S.%f"),
                    start_date=dataStartTime.strftime("%Y-%m-%d"),
                    numSamples=numSamples,
                    sample_freq=dataFs,
                    lsb_applied=True)
                writer.printInfo()