def convertDataset(group, datasetName, newGroup, newDatasetName):
''' Converts a sensor group into the L1B format; option to change dataset name.
Moves dataset to new group.
The separate DATETAG, TIMETAG2, and DATETIME datasets are combined into
the sensor dataset. This also adds a temporary column in the sensor data
array for datetime to be used in interpolation. This is later removed, as
HDF5 does not support datetime. '''
dataset = group.getDataset(datasetName)
dateData = group.getDataset("DATETAG")
timeData = group.getDataset("TIMETAG2")
dateTimeData = group.getDataset("DATETIME")
# Convert degrees minutes to decimal degrees format; only for GPS, not ANCILLARY_METADATA
if group.id.startswith("GP"):
if newDatasetName == "LATITUDE":
latPosData = group.getDataset("LATPOS")
latHemiData = group.getDataset("LATHEMI")
for i in range(dataset.data.shape[0]):
latDM = latPosData.data["NONE"][i]
latDirection = latHemiData.data["NONE"][i]
latDD = Utilities.dmToDd(latDM, latDirection)
latPosData.data["NONE"][i] = latDD
if newDatasetName == "LONGITUDE":
lonPosData = group.getDataset("LONPOS")
lonHemiData = group.getDataset("LONHEMI")
for i in range(dataset.data.shape[0]):
lonDM = lonPosData.data["NONE"][i]
lonDirection = lonHemiData.data["NONE"][i]
lonDD = Utilities.dmToDd(lonDM, lonDirection)
lonPosData.data["NONE"][i] = lonDD
newSensorData = newGroup.addDataset(newDatasetName)
# Datetag, Timetag2, and Datetime columns added to sensor data array
newSensorData.columns["Datetag"] = dateData.data["NONE"].tolist()
newSensorData.columns["Timetag2"] = timeData.data["NONE"].tolist()
newSensorData.columns["Datetime"] = dateTimeData.data
# Copies over the sensor dataset from original group to newGroup
for k in dataset.data.dtype.names: # For each waveband (or vector data for other groups)
#print("type",type(esData.data[k]))
newSensorData.columns[k] = dataset.data[k].tolist()
newSensorData.columnsToDataset()
def interpolateGPSData(node, gpsGroup):
print("Interpolate GPS Data")
if gpsGroup is None:
print("WARNING, gpsGroup is None")
return
refGroup = node.getGroup("Reference")
esData = refGroup.getDataset("ES_hyperspectral")
# GPS
# Creates new gps group with Datetag/Timetag2 columns appended to all datasets
gpsTimeData = gpsGroup.getDataset("UTCPOS")
gpsCourseData = gpsGroup.getDataset("COURSE")
gpsLatPosData = gpsGroup.getDataset("LATPOS")
gpsLonPosData = gpsGroup.getDataset("LONPOS")
gpsMagVarData = gpsGroup.getDataset("MAGVAR")
gpsSpeedData = gpsGroup.getDataset("SPEED")
gpsLatHemiData = gpsGroup.getDataset("LATHEMI")
gpsLonHemiData = gpsGroup.getDataset("LONHEMI")
newGPSGroup = node.getGroup("GPS")
newGPSCourseData = newGPSGroup.addDataset("COURSE")
newGPSLatPosData = newGPSGroup.addDataset("LATPOS")
newGPSLonPosData = newGPSGroup.addDataset("LONPOS")
newGPSMagVarData = newGPSGroup.addDataset("MAGVAR")
newGPSSpeedData = newGPSGroup.addDataset("SPEED")
# Add Datetag, Timetag2 data to gps groups
# This matches ES data after interpolation
newGPSCourseData.columns["Datetag"] = esData.data["Datetag"].tolist()
newGPSCourseData.columns["Timetag2"] = esData.data["Timetag2"].tolist()
newGPSLatPosData.columns["Datetag"] = esData.data["Datetag"].tolist()
newGPSLatPosData.columns["Timetag2"] = esData.data["Timetag2"].tolist()
newGPSLonPosData.columns["Datetag"] = esData.data["Datetag"].tolist()
newGPSLonPosData.columns["Timetag2"] = esData.data["Timetag2"].tolist()
newGPSMagVarData.columns["Datetag"] = esData.data["Datetag"].tolist()
newGPSMagVarData.columns["Timetag2"] = esData.data["Timetag2"].tolist()
newGPSSpeedData.columns["Datetag"] = esData.data["Datetag"].tolist()
newGPSSpeedData.columns["Timetag2"] = esData.data["Timetag2"].tolist()
x = []
y = []
# Convert degrees minutes to decimal degrees format
for i in range(gpsTimeData.data.shape[0]):
latDM = gpsLatPosData.data["NONE"][i]
latDirection = gpsLatHemiData.data["NONE"][i]
latDD = Utilities.dmToDd(latDM, latDirection)
gpsLatPosData.data["NONE"][i] = latDD
lonDM = gpsLonPosData.data["NONE"][i]
lonDirection = gpsLonHemiData.data["NONE"][i]
lonDD = Utilities.dmToDd(lonDM, lonDirection)
gpsLonPosData.data["NONE"][i] = lonDD
x.append(lonDD)
y.append(latDD)
#print("PlotGPS")
#Utilities.plotGPS(x, y, 'test1')
#print("PlotGPS - DONE")
# Convert GPS UTC time values to seconds to be used for interpolation
xTimer = []
for i in range(gpsTimeData.data.shape[0]):
xTimer.append(Utilities.utcToSec(gpsTimeData.data["NONE"][i]))
# Convert ES TimeTag2 values to seconds to be used for interpolation
yTimer = []
for i in range(esData.data.shape[0]):
yTimer.append(Utilities.timeTag2ToSec(esData.data["Timetag2"][i]))
# Interpolate by time values
ProcessL2s.interpolateL2s(gpsCourseData, xTimer, yTimer,
newGPSCourseData, 'linear')
ProcessL2s.interpolateL2s(gpsLatPosData, xTimer, yTimer,
newGPSLatPosData, 'linear')
ProcessL2s.interpolateL2s(gpsLonPosData, xTimer, yTimer,
newGPSLonPosData, 'linear')
ProcessL2s.interpolateL2s(gpsMagVarData, xTimer, yTimer,
newGPSMagVarData, 'linear')
ProcessL2s.interpolateL2s(gpsSpeedData, xTimer, yTimer,
newGPSSpeedData, 'linear')
newGPSCourseData.columnsToDataset()
newGPSLatPosData.columnsToDataset()
newGPSLonPosData.columnsToDataset()
newGPSMagVarData.columnsToDataset()
newGPSSpeedData.columnsToDataset()
def processSplitLonFlag(filepath, dataDir, calibrationMap, startLongitude,
endLongitude, direction):
print("Read: " + filepath)
# gpsState: set to 1 if within start/end longitude, else 0
gpsState = 0
# Saves the starting and ending GPS HDF Group
gpGPSStart = None
gpGPSEnd = None
# Saves the raw file header
header = b""
#msg = b""
#message = b""
# Index of start/end message block for start/end longitude
iStart = -1
iEnd = -1
#(dirpath, filename) = os.path.split(filepath)
#print(filename)
#posframe = 1
# Note: Prosoft adds posframe=1 to the GPS for some reason
#print(contextMap.keys())
#gpsGroup = contextMap["$GPRMC"]
#ds = gpsGroup.getDataset("POSFRAME")
#ds.appendColumn(u"COUNT", posframe)
#posframe += 1
# Start reading raw binary file
with open(filepath, 'rb') as f:
while 1:
# Reads a block of binary data from file
pos = f.tell()
b = f.read(PreprocessRawFile.MAX_TAG_READ)
f.seek(pos)
if not b:
break
#print b
# Search for frame tag string in block
for i in range(0, PreprocessRawFile.MAX_TAG_READ):
testString = b[i:].upper()
#print("test: ", testString[:6])
# Reset file position on max read (frame tag not found)
if i == PreprocessRawFile.MAX_TAG_READ - 1:
#f.read(PreprocessRawFile.MAX_TAG_READ)
f.read(PreprocessRawFile.RESET_TAG_READ)
break
# Reads header message type from frame tag
if testString.startswith(b"SATHDR"):
#print("SATHDR")
if i > 0:
f.read(i)
header += f.read(PreprocessRawFile.SATHDR_READ)
break
# Reads messages that starts with \$GPRMC and retrieves the CalibrationFile from map
else:
bytesRead = 0
for key in calibrationMap:
cf = calibrationMap[key]
if testString.startswith(
b"$GPRMC") and testString.startswith(
cf.id.upper().encode("utf-8")):
if i > 0:
f.read(i)
# Read block from start of message
pos = f.tell()
msg = f.read(PreprocessRawFile.MAX_BLOCK_READ)
f.seek(pos)
# Convert message to HDFGroup
gp = HDFGroup()
bytesRead = cf.convertRaw(msg, gp)
# Read till end of message
if bytesRead >= 0:
f.read(bytesRead)
#if gpsState == 0:
# msg = f.read(bytesRead)
#else:
# msg += f.read(bytesRead)
#gp.printd()
if gp.getDataset("LONPOS"):
#print("has gps")
lonData = gp.getDataset("LONPOS")
lonHemiData = gp.getDataset("LONHEMI")
lonDM = lonData.columns["NONE"][0]
lonDirection = lonHemiData.columns["NONE"][
0]
longitude = Utilities.dmToDd(
lonDM, lonDirection)
#print(longitude)
# Detect if we are in specified longitude
if longitude > startLongitude and longitude < endLongitude:
if gpsState == 0:
iStart = pos
gpGPSStart = gp
else:
iEnd = f.tell()
gpGPSEnd = gp
#message += msg
gpsState = 1
# Not within start/end longitude
else:
if gpsState == 1:
#print("Test")
PreprocessRawFile.createRawFile(
dataDir, gpGPSStart, gpGPSEnd,
direction, f, header, iStart,
iEnd)
gpsState = 0
break
if bytesRead > 0:
break
# In case file finished processing without reaching endLongitude
if gpsState == 1:
if gpGPSStart is not None and gpGPSEnd is not None:
PreprocessRawFile.createRawFile(dataDir, gpGPSStart,
gpGPSEnd, direction, f,
header, iStart, iEnd)
