def addGroup(self, name):
gp = self.getGroup(name)
if not gp:
gp = HDFGroup()
gp.id = name
self.groups.append(gp)
return gp
def readHDF5(fp):
root = HDFRoot()
with h5py.File(fp, "r") as f:
# set name to text after last '/'
name = f.name[f.name.rfind("/")+1:]
if len(name) == 0:
name = "/"
root.id = name
# Read attributes
#print("Attributes:", [k for k in f.attrs.keys()])
for k in f.attrs.keys():
root.attributes[k] = f.attrs[k].decode("utf-8")
# Use the following when using h5toh4 converter:
#root.attributes[k.replace("__GLOSDS", "")] = f.attrs[k].decode("utf-8")
# Read groups
for k in f.keys():
item = f.get(k)
#print(item)
if isinstance(item, h5py.Group):
gp = HDFGroup()
root.groups.append(gp)
gp.read(item)
elif isinstance(item, h5py.Dataset):
print("HDFRoot should not contain datasets")
return root
def processL1a(fp, calibrationMap):
'''
Reads a raw binary file and generates a L1a HDF5 file
'''
(_, fileName) = os.path.split(fp)
# Generate root attributes
root = HDFRoot()
root.id = "/"
root.attributes["HYPERINSPACE"] = MainConfig.settings["version"]
root.attributes["CAL_FILE_NAMES"] = ','.join(calibrationMap.keys())
root.attributes["WAVELENGTH_UNITS"] = "nm"
root.attributes["LI_UNITS"] = "count"
root.attributes["LT_UNITS"] = "count"
root.attributes["ES_UNITS"] = "count"
root.attributes["SATPYR_UNITS"] = "count"
root.attributes["RAW_FILE_NAME"] = fileName
root.attributes["PROCESSING_LEVEL"] = "1a"
now = dt.datetime.now()
timestr = now.strftime("%d-%b-%Y %H:%M:%S")
root.attributes["FILE_CREATION_TIME"] = timestr
# SZA Filter configuration parameter added to attributes below
msg = f"ProcessL1a.processL1a: {timestr}"
print(msg)
Utilities.writeLogFile(msg)
contextMap = collections.OrderedDict()
for key in calibrationMap:
cf = calibrationMap[key]
gp = HDFGroup()
gp.id = cf.instrumentType
contextMap[cf.id] = gp
# print("contextMap:", list(contextMap.keys()))
# print("calibrationMap:", list(calibrationMap.keys()))
print('Reading in raw binary data may take a moment.')
RawFileReader.readRawFile(fp, calibrationMap, contextMap, root)
# Populate HDF group attributes
for key in calibrationMap:
cf = calibrationMap[key]
gp = contextMap[cf.id]
# Don't add contexts that did not match any data in RawFileReader
if 'CalFileName' not in gp.attributes:
continue
gp.attributes["InstrumentType"] = cf.instrumentType
gp.attributes["Media"] = cf.media
gp.attributes["MeasMode"] = cf.measMode
gp.attributes["FrameType"] = cf.frameType
gp.getTableHeader(cf.sensorType)
gp.attributes[
"DISTANCE_1"] = "Pressure " + cf.sensorType + " 1 1 0"
gp.attributes["DISTANCE_2"] = "Surface " + cf.sensorType + " 1 1 0"
# gp.attributes["SensorDataList"] = ", ".join([x for x in gp.datasets.keys()])
gp.attributes["SensorDataList"] = ", ".join(
list(gp.datasets.keys()))
if gp.id != 'SAS' and gp.id != 'Reference':
root.groups.append(gp)
# Insure essential data groups are present before proceeding
hld = 0
hsl = 0
hse = 0
hed = 0
gps = 0
for gp in root.groups:
if gp.id.startswith("HLD"):
hld += 1
if gp.id.startswith("HSL"):
hsl += 1
if gp.id.startswith("HSE"):
hse += 1
if gp.id.startswith("HED"):
hed += 1
if gp.id.startswith("GP"):
gps += 1
if hld != 2 or hsl != 2 or hse != 1 or hed != 1 or gps != 1:
msg = "ProcessL1a.processL1a: Essential dataset missing. Check your configuration calibration files match cruise setup. Aborting."
msg = f'{msg}\ngps: {gps} :1'
msg = f'{msg}\nhed: {hed} :1'
msg = f'{msg}\nhld: {hld} :2'
msg = f'{msg}\nhse: {hse} :1'
msg = f'{msg}\nhsl: {hsl} :2'
print(msg)
Utilities.writeLogFile(msg)
return None
# Update the GPS group to include a datasets for DATETAG and TIMETAG2
for gp in root.groups:
if gp.id.startswith("GP"):
gpsGroup = gp
# Need year-gang and sometimes Datetag from one of the sensors
if gp.id.startswith("HSE"):
esDateTag = gp.datasets["DATETAG"].columns["NONE"]
esTimeTag2 = gp.datasets["TIMETAG2"].columns["NONE"]
esSec = []
for time in esTimeTag2:
esSec.append(Utilities.timeTag2ToSec(time))
gpsGroup.addDataset("DATETAG")
gpsGroup.addDataset("TIMETAG2")
if "UTCPOS" in gpsGroup.datasets:
gpsTime = gpsGroup.datasets["UTCPOS"].columns["NONE"]
elif "TIME" in gpsGroup.datasets:
# prepSAS output
gpsTime = gpsGroup.datasets["TIME"].columns["UTC"]
else:
msg = 'Failed to import GPS data.'
print(msg)
Utilities.writeLogFile(msg)
return None
# Another case for GPGGA input...
if gpsGroup.id.startswith("GPGGA"):
# No date is provided in GPGGA, need to find nearest time in Es and take the Datetag from Es
''' Catch-22. In order to covert the gps time, we need the year and day, which GPGGA does not have.
To get these, could compare to find the nearest DATETAG in Es. In order to compare the gps time
to the Es time to find the nearest, I would need to convert them to datetimes ... which would
require the year and day. Instead, I use either the first or last Datetag from Es, depending
on whether UTC 00:00 was crossed.'''
# If the date does not change in Es, then no problem, use the Datetag of Es first element.
# Otherwise, change the datetag at midnight by one day
gpsDateTag = []
gpsTimeTag2 = []
if esDateTag[0] != esDateTag[-1]:
msg = "ProcessL1a.processL1a: Warning: File crosses UTC 00:00. Adjusting timestamps for matchup of Datetag."
print(msg)
Utilities.writeLogFile(msg)
newDay = False
for time in gpsTime:
gpsSec = Utilities.utcToSec(time)
if not 'gpsSecPrior' in locals():
gpsSecPrior = gpsSec
# Test for a change of ~24 hrs between this sample and the last sample
# To cross 0, gpsSecPrior would need to be approaching 86400 seconds
# In that case, take the final Es Datetag
if (gpsSecPrior - gpsSec) > 86000:
# Once triggered the first time, this will remain true for remainder of file
newDay = True
if newDay is True:
gpsDateTag.append(esDateTag[-1])
dtDate = Utilities.dateTagToDateTime(esDateTag[-1])
gpsTimeTag2.append(
Utilities.datetime2TimeTag2(
Utilities.utcToDateTime(dtDate, time)))
else:
gpsDateTag.append(esDateTag[0])
dtDate = Utilities.dateTagToDateTime(esDateTag[0])
gpsTimeTag2.append(
Utilities.datetime2TimeTag2(
Utilities.utcToDateTime(dtDate, time)))
gpsSecPrior = gpsSec
else:
for time in gpsTime:
gpsDateTag.append(esDateTag[0])
dtDate = Utilities.dateTagToDateTime(esDateTag[0])
gpsTimeTag2.append(
Utilities.datetime2TimeTag2(
Utilities.utcToDateTime(dtDate, time)))
gpsGroup.datasets["DATETAG"].columns["NONE"] = gpsDateTag
gpsGroup.datasets["TIMETAG2"].columns["NONE"] = gpsTimeTag2
# Converts gp.columns to numpy array
for gp in root.groups:
if gp.id.startswith(
"SATMSG"): # Don't convert these strings to datasets.
for ds in gp.datasets.values():
ds.columnsToDataset()
else:
for ds in gp.datasets.values():
if not ds.columnsToDataset():
msg = "ProcessL1a.processL1a: Essential column cannot be converted to Dataset. Aborting."
print(msg)
Utilities.writeLogFile(msg)
return None
# Apply SZA filter; Currently only works with SolarTracker data at L1A (again possible in L2)
if ConfigFile.settings["bL1aCleanSZA"]:
root.attributes['SZA_FILTER_L1A'] = ConfigFile.settings[
"fL1aCleanSZAMax"]
for gp in root.groups:
# try:
if 'FrameTag' in gp.attributes:
if gp.attributes["FrameTag"].startswith("SATNAV"):
elevData = gp.getDataset("ELEVATION")
elevation = elevData.data.tolist()
szaLimit = float(
ConfigFile.settings["fL1aCleanSZAMax"])
''' It would be good to add local time as a printed output with SZA'''
if (90 - np.nanmax(elevation)) > szaLimit:
msg = f'SZA too low. Discarding entire file. {round(90-np.nanmax(elevation))}'
print(msg)
Utilities.writeLogFile(msg)
return None
else:
msg = f'SZA passed filter: {round(90-np.nanmax(elevation))}'
print(msg)
Utilities.writeLogFile(msg)
else:
print(f'No FrameTag in {gp.id} group')
return root
def readLUT(node, filename, headerlines):
"""
Read in the M99 LUT for 550 nm
Required arguments:
node = HDF5 Root object
filename = name of SeaBASS input file (string)
headerlines = # of headerlines in the LUT text file
LUT is in self.groups[0].datasets['LUT'].data
phiView is the relative azimuth angle, and phiSun is 180 - phiView
"""
node.id = 'rhoLUT'
node.attributes['LUT origin'] = filename.split(os.path.sep)[-1]
node.attributes[
'Citation'] = 'Mobley, 1999, Appl Opt 38, page 7445, Eqn. 4'
try:
fileobj = open(filename, 'r')
except Exception as e:
print(f'Unable to open file for reading: {filename}. Error: {e}')
return
try:
lines = fileobj.readlines()
fileobj.close()
except Exception as e:
print(f'Unable to read data from file: {filename}. Error: {e}')
return
# Create the dictionary LUT
lut = {
'wind': [],
'sza': [],
'theta': [],
'phiSun': [],
'phiView': [],
'rho': []
}
# Later, interpolate or find-nearest the field data to these steps to match...
wind = [] # 0:2:14
sza = [] # 0:10:80
# I =[] # 1:1:10
# J = [] # 1:1:13
theta = [] # 0:10:80, 87.5
phiSun = [] # 0:15:180 # phi of 45 deg is relaz of 135 deg
phiView = [] # 0:15:180
rho = [] # result
for i, line in enumerate(lines):
if i < headerlines:
continue
data_row = re.search("WIND SPEED =", line)
data_row = re.search("THETA_SUN", line)
if data_row:
elems = re.findall('\d+\.\d+', line)
continue
linefloat = [float(elem) for elem in line.split()]
wind.append(float(elems[0]))
sza.append(float(elems[1]))
theta.append(linefloat[2])
phiSun.append(linefloat[3])
phiView.append(linefloat[4])
rho.append(linefloat[5])
lut['wind'] = wind
lut['sza'] = sza
lut['theta'] = theta
lut['phiSun'] = phiSun
lut['phiView'] = phiView
lut['rho'] = rho
gp = HDFGroup()
gp.id = 'LUT'
node.groups.append(gp)
LUTDataset = gp.addDataset("LUT")
LUTDataset.columns = lut
LUTDataset.columnsToDataset()
return node
def processSplitLonFlag(filepath, dataDir, calibrationMap, startLongitude,
endLongitude, direction):
print("Read: " + filepath)
# gpsState: set to 1 if within start/end longitude, else 0
gpsState = 0
# Saves the starting and ending GPS HDF Group
gpGPSStart = None
gpGPSEnd = None
# Saves the raw file header
header = b""
#msg = b""
#message = b""
# Index of start/end message block for start/end longitude
iStart = -1
iEnd = -1
#(dirpath, filename) = os.path.split(filepath)
#print(filename)
#posframe = 1
# Note: Prosoft adds posframe=1 to the GPS for some reason
#print(contextMap.keys())
#gpsGroup = contextMap["$GPRMC"]
#ds = gpsGroup.getDataset("POSFRAME")
#ds.appendColumn(u"COUNT", posframe)
#posframe += 1
# Start reading raw binary file
with open(filepath, 'rb') as f:
while 1:
# Reads a block of binary data from file
pos = f.tell()
b = f.read(PreprocessRawFile.MAX_TAG_READ)
f.seek(pos)
if not b:
break
#print b
# Search for frame tag string in block
for i in range(0, PreprocessRawFile.MAX_TAG_READ):
testString = b[i:].upper()
#print("test: ", testString[:6])
# Reset file position on max read (frame tag not found)
if i == PreprocessRawFile.MAX_TAG_READ - 1:
#f.read(PreprocessRawFile.MAX_TAG_READ)
f.read(PreprocessRawFile.RESET_TAG_READ)
break
# Reads header message type from frame tag
if testString.startswith(b"SATHDR"):
#print("SATHDR")
if i > 0:
f.read(i)
header += f.read(PreprocessRawFile.SATHDR_READ)
break
# Reads messages that starts with \$GPRMC and retrieves the CalibrationFile from map
else:
bytesRead = 0
for key in calibrationMap:
cf = calibrationMap[key]
if testString.startswith(
b"$GPRMC") and testString.startswith(
cf.id.upper().encode("utf-8")):
if i > 0:
f.read(i)
# Read block from start of message
pos = f.tell()
msg = f.read(PreprocessRawFile.MAX_BLOCK_READ)
f.seek(pos)
# Convert message to HDFGroup
gp = HDFGroup()
bytesRead = cf.convertRaw(msg, gp)
# Read till end of message
if bytesRead >= 0:
f.read(bytesRead)
#if gpsState == 0:
# msg = f.read(bytesRead)
#else:
# msg += f.read(bytesRead)
#gp.printd()
if gp.getDataset("LONPOS"):
#print("has gps")
lonData = gp.getDataset("LONPOS")
lonHemiData = gp.getDataset("LONHEMI")
lonDM = lonData.columns["NONE"][0]
lonDirection = lonHemiData.columns["NONE"][
0]
longitude = Utilities.dmToDd(
lonDM, lonDirection)
#print(longitude)
# Detect if we are in specified longitude
if longitude > startLongitude and longitude < endLongitude:
if gpsState == 0:
iStart = pos
gpGPSStart = gp
else:
iEnd = f.tell()
gpGPSEnd = gp
#message += msg
gpsState = 1
# Not within start/end longitude
else:
if gpsState == 1:
#print("Test")
PreprocessRawFile.createRawFile(
dataDir, gpGPSStart, gpGPSEnd,
direction, f, header, iStart,
iEnd)
gpsState = 0
break
if bytesRead > 0:
break
# In case file finished processing without reaching endLongitude
if gpsState == 1:
if gpGPSStart is not None and gpGPSEnd is not None:
PreprocessRawFile.createRawFile(dataDir, gpGPSStart,
gpGPSEnd, direction, f,
header, iStart, iEnd)
def processSplitTimer(filepath, dataDir, calibrationMap):
print("Split Timer: " + filepath)
# Used to detect when the timer is reset
saveTime = -1
# Saves the starting and ending GPS HDF Group
timerStart = None
timerEnd = None
# Saves the raw file header
header = b""
# Index of start/end message block for start/end timer
iStart = -1
iEnd = -1
# Start reading raw binary file
with open(filepath, 'rb') as f:
while 1:
# Reads a block of binary data from file
pos = f.tell()
b = f.read(PreprocessRawFile.MAX_TAG_READ)
f.seek(pos)
if not b:
break
#print b
# Search for frame tag string in block
for i in range(0, PreprocessRawFile.MAX_TAG_READ):
testString = b[i:].upper()
# Reset file position on max read (frame tag not found)
if i == PreprocessRawFile.MAX_TAG_READ - 1:
f.read(PreprocessRawFile.RESET_TAG_READ)
break
# Reads header message type from frame tag
if testString.startswith(b"SATHDR"):
#print("SATHDR")
if i > 0:
f.read(i)
header += f.read(PreprocessRawFile.SATHDR_READ)
break
# Reads messages that starts with \$GPRMC and retrieves the CalibrationFile from map
else:
bytesRead = 0
for key in calibrationMap:
cf = calibrationMap[key]
if testString.startswith(
cf.id.upper().encode("utf-8")):
if i > 0:
f.read(i)
# Read block from start of message
pos = f.tell()
msg = f.read(PreprocessRawFile.MAX_BLOCK_READ)
f.seek(pos)
# Convert message to HDFGroup
gp = HDFGroup()
bytesRead = cf.convertRaw(msg, gp)
# Read till end of message
if bytesRead >= 0:
f.read(bytesRead)
#gp.printd()
if gp.getDataset("ES") and gp.getDataset(
"TIMER"):
#print("has gps")
timerData = gp.getDataset("TIMER")
time = timerData.columns["NONE"][0]
#print(time)
# Detect if we are in specified longitude
if time < saveTime:
PreprocessRawFile.createRawFileTimer(
dataDir, timerStart, timerEnd, f,
header, iStart, iEnd)
timerStart = gp
iStart = pos
saveTime = time
else:
if saveTime == -1:
timerStart = gp
iStart = pos
iEnd = f.tell()
timerEnd = gp
saveTime = time
break
if bytesRead > 0:
break
def cleanSunAngle(filepath,
calibrationMap,
angleMin,
angleMax,
rotatorHomeAngle=0):
print("Clean Raw File")
header = b""
message = b""
# Index of start/end message block for start/end longitude
iStart = 0
iEnd = 0
# Start reading raw binary file
with open(filepath, 'rb') as f:
while 1:
# Reads a block of binary data from file
pos = f.tell()
b = f.read(PreprocessRawFile.MAX_TAG_READ)
f.seek(pos)
if not b:
break
#print b
# Search for frame tag string in block
for i in range(0, PreprocessRawFile.MAX_TAG_READ):
testString = b[i:].upper()
#print("test: ", testString[:6])
# Reset file position on max read (frame tag not found)
if i == PreprocessRawFile.MAX_TAG_READ - 1:
#f.read(PreprocessRawFile.MAX_TAG_READ)
f.read(PreprocessRawFile.RESET_TAG_READ)
break
# Reads header message type from frame tag
if testString.startswith(b"SATHDR"):
#print("SATHDR")
if i > 0:
f.read(i)
header += f.read(PreprocessRawFile.SATHDR_READ)
break
# Reads messages that starts with SATNAV and retrieves the CalibrationFile from map
else:
bytesRead = 0
for key in calibrationMap:
cf = calibrationMap[key]
if testString.startswith(
b"SATNAV") and testString.startswith(
cf.id.upper().encode("utf-8")):
if i > 0:
f.read(i)
# Read block from start of message
pos = f.tell()
msg = f.read(PreprocessRawFile.MAX_BLOCK_READ)
f.seek(pos)
# Convert message to HDFGroup
gp = HDFGroup()
bytesRead = cf.convertRaw(msg, gp)
# Read till end of message
if bytesRead >= 0:
f.read(bytesRead)
#gp.printd()
if gp.getDataset("AZIMUTH") and gp.getDataset(
"HEADING") and gp.getDataset(
"POINTING"):
angle = 0
azimuthData = gp.getDataset("AZIMUTH")
azimuth = azimuthData.columns["SUN"][0]
#headingData = gp.getDataset("HEADING")
#sasTrue = headingData.columns["SAS_TRUE"][0]
#angle = PreprocessRawFile.normalizeAngle(azimuth - sasTrue)
#print("azimuth - sasTrue: ", PreprocessRawFile.normalizeAngle(azimuth - sasTrue))
pointingData = gp.getDataset("POINTING")
rotatorAngle = pointingData.columns[
"ROTATOR"][0]
headingData = gp.getDataset("HEADING")
shipTrue = headingData.columns[
"SHIP_TRUE"][0]
#angle = PreprocessRawFile.normalizeAngle(shipTrue + rotatorHomeAngle)
#angle = PreprocessRawFile.normalizeAngle(angle - rotatorAngle)
#angle = PreprocessRawFile.normalizeAngle(azimuth - angle)
angle = PreprocessRawFile.normalizeAngle(
rotatorAngle + shipTrue - azimuth +
270 - 35 + rotatorHomeAngle)
#print("Angle: ", angle)
#print("Angle: ", angle)
#if angle >= 90 and angle <= 135:
if angle >= angleMin and angle <= angleMax:
# iStart is -1 if previous SATNAV was bad angle
if iStart != -1:
# Append measurements from previous block to new file
iEnd = f.tell()
pos = f.tell()
f.seek(iStart)
msg = f.read(iEnd - iStart)
f.seek(pos)
message += msg
iStart = f.tell()
else:
# Reset iStart to start of current SATNAV
iStart = f.tell() - bytesRead
else:
# Found bad angles, set iStart to -1 to ignore raw data
print("Skip Bad Angle: ", angle)
iStart = -1
break
if bytesRead > 0:
break
if iStart != -1:
f.seek(iStart)
message += f.read()
# Write file
with open(filepath, 'wb') as fout:
fout.write(message)
def cleanRotatorAngle(filepath,
calibrationMap,
angleMin,
angleMax,
rotatorHomeAngle=0,
rotatorDelay=60):
print("Clean Raw File")
header = b""
message = b""
# Index of start/end message block for start/end longitude
iStart = 0
iEnd = 0
init = False
saveTime = 0
saveAngle = 0
# Start reading raw binary file
with open(filepath, 'rb') as f:
while 1:
# Reads a block of binary data from file
pos = f.tell()
b = f.read(PreprocessRawFile.MAX_TAG_READ)
f.seek(pos)
if not b:
break
#print b
# Search for frame tag string in block
for i in range(0, PreprocessRawFile.MAX_TAG_READ):
testString = b[i:].upper()
#print("test: ", testString[:6])
# Reset file position on max read (frame tag not found)
if i == PreprocessRawFile.MAX_TAG_READ - 1:
#f.read(PreprocessRawFile.MAX_TAG_READ)
f.read(PreprocessRawFile.RESET_TAG_READ)
break
# Reads header message type from frame tag
if testString.startswith(b"SATHDR"):
#print("SATHDR")
if i > 0:
f.read(i)
header += f.read(PreprocessRawFile.SATHDR_READ)
break
# Reads messages that starts with SATNAV and retrieves the CalibrationFile from map
else:
bytesRead = 0
for key in calibrationMap:
cf = calibrationMap[key]
if testString.startswith(
b"SATNAV") and testString.startswith(
cf.id.upper().encode("utf-8")):
if i > 0:
f.read(i)
# Read block from start of message
pos = f.tell()
msg = f.read(PreprocessRawFile.MAX_BLOCK_READ)
f.seek(pos)
# Convert message to HDFGroup
gp = HDFGroup()
bytesRead = cf.convertRaw(msg, gp)
# Read till end of message
if bytesRead >= 0:
f.read(bytesRead)
#gp.printd()
if gp.getDataset("AZIMUTH") and gp.getDataset(
"HEADING") and gp.getDataset(
"POINTING"):
pointingData = gp.getDataset("POINTING")
#angle = pointingData.columns["ROTATOR"][0] - rotatorHomeAngle
angle = pointingData.columns["ROTATOR"][0]
timetag2Data = gp.getDataset("TIMETAG2")
time = Utilities.timeTag2ToSec(
timetag2Data.columns["NONE"][0])
if not init:
saveAngle = angle
saveTime = time
init = True
else:
# Detect angle changed
if saveAngle < angle + 0.05 and saveAngle > angle - 0.05:
saveAngle = angle
else:
#print("Angle:", saveAngle, angle, "Time:", time)
saveAngle = angle
saveTime = time
# Tell program to ignore angles until rotatorDelay time
angle = angleMin - 1
#print("Angle: ", angle)
#if angle >= 90 and angle <= 135:
#if angle >= angleMin and angle <= angleMax:
if angle >= angleMin and angle <= angleMax and time > saveTime + rotatorDelay:
# iStart is -1 if previous SATNAV was bad angle
if iStart != -1:
# Append measurements from previous block to new file
iEnd = f.tell()
pos = f.tell()
f.seek(iStart)
msg = f.read(iEnd - iStart)
f.seek(pos)
message += msg
iStart = f.tell()
else:
# Reset iStart to start of current SATNAV
iStart = f.tell() - bytesRead
else:
# Found bad angles, set iStart to -1 to ignore raw data
if time < saveTime + rotatorDelay:
print(
"Rotator Angle Changed, Time: ",
time)
else:
print("Skip Rotator Angle: ",
angle)
iStart = -1
break
if bytesRead > 0:
break
if iStart != -1:
f.seek(iStart)
message += f.read()
# Write file
with open(filepath, 'wb') as fout:
fout.write(message)