def QC(node, modRoot):
''' Add model data. QC for wind, Lt, SZA, spectral outliers, and met filters'''
print(
"Add model data. QC for wind, Lt, SZA, spectral outliers, and met filters"
)
referenceGroup = node.getGroup("IRRADIANCE")
sasGroup = node.getGroup("RADIANCE")
gpsGroup = node.getGroup('GPS')
satnavGroup = None
ancGroup = None
pyrGroup = None
for gp in node.groups:
if gp.id.startswith("SOLARTRACKER"):
if gp.id != "SOLARTRACKER_STATUS":
satnavGroup = gp
if gp.id.startswith("ANCILLARY"):
ancGroup = gp
ancGroup.id = "ANCILLARY" # shift back from ANCILLARY_METADATA
if gp.id.startswith("PYROMETER"):
pyrGroup = gp
# Regardless of whether SolarTracker/pySAS is used, Ancillary data will have been already been
# interpolated in L1B as long as the ancillary file was read in at L1AQC. Regardless, these need
# to have model data and/or default values incorporated.
# If GMAO modeled data is selected in ConfigWindow, and an ancillary field data file
# is provided in Main Window, then use the model data to fill in gaps in the field
# record. Otherwise, use the selected default values from ConfigWindow
# This step is only necessary for the ancillary datasets that REQUIRE
# either field or GMAO or GUI default values. The remaining ancillary data
# are culled from datasets in groups in L1B
ProcessL1bqc.includeModelDefaults(ancGroup, modRoot)
# Shift metadata into the ANCILLARY group as needed (i.e. from GPS).
#
# GPS Group
# These have TT2/Datetag incorporated in arrays
# Change their column names from NONE to something appropriate to be consistent in
# ancillary group going forward.
# Replace metadata lat/long with GPS lat/long, in case the former is from the ancillary file
ancGroup.datasets['LATITUDE'] = gpsGroup.getDataset('LATITUDE')
ancGroup.datasets['LATITUDE'].changeColName('NONE', 'LATITUDE')
ancGroup.datasets['LONGITUDE'] = gpsGroup.getDataset('LONGITUDE')
ancGroup.datasets['LONGITUDE'].changeColName('NONE', 'LONGITUDE')
# Take Heading and Speed preferentially from GPS
if 'HEADING' in gpsGroup.datasets:
# These have TT2/Datetag incorporated in arrays
ancGroup.addDataset('HEADING')
ancGroup.datasets['HEADING'] = gpsGroup.getDataset('COURSE')
ancGroup.datasets['HEADING'].changeColName('TRUE', 'HEADING')
if 'SOG' in gpsGroup.datasets:
ancGroup.addDataset('SOG')
ancGroup.datasets['SOG'] = gpsGroup.getDataset('SOG')
ancGroup.datasets['SOG'].changeColName('NONE', 'SOG')
if 'HEADING' not in gpsGroup.datasets and 'HEADING' in ancGroup.datasets:
ancGroup.addDataset('HEADING')
# ancGroup.datasets['HEADING'] = ancTemp.getDataset('HEADING')
ancGroup.datasets['HEADING'] = ancGroup.getDataset('HEADING')
ancGroup.datasets['HEADING'].changeColName('NONE', 'HEADING')
if 'SOG' not in gpsGroup.datasets and 'SOG' in ancGroup.datasets:
ancGroup.datasets['SOG'] = ancGroup.getDataset('SOG')
ancGroup.datasets['SOG'].changeColName('NONE', 'SOG')
if 'SPEED_F_W' in ancGroup.datasets:
ancGroup.addDataset('SPEED_F_W')
ancGroup.datasets['SPEED_F_W'] = ancGroup.getDataset('SPEED_F_W')
ancGroup.datasets['SPEED_F_W'].changeColName('NONE', 'SPEED_F_W')
# Take SZA and SOLAR_AZ preferentially from ancGroup (calculated with pysolar in L1C)
ancGroup.datasets['SZA'].changeColName('NONE', 'SZA')
ancGroup.datasets['SOLAR_AZ'].changeColName('NONE', 'SOLAR_AZ')
if 'CLOUD' in ancGroup.datasets:
ancGroup.datasets['CLOUD'].changeColName('NONE', 'CLOUD')
if 'PITCH' in ancGroup.datasets:
ancGroup.datasets['PITCH'].changeColName('NONE', 'PITCH')
if 'ROLL' in ancGroup.datasets:
ancGroup.datasets['ROLL'].changeColName('NONE', 'ROLL')
if 'STATION' in ancGroup.datasets:
ancGroup.datasets['STATION'].changeColName('NONE', 'STATION')
if 'WAVE_HT' in ancGroup.datasets:
ancGroup.datasets['WAVE_HT'].changeColName('NONE', 'WAVE_HT')
if 'SALINITY' in ancGroup.datasets:
ancGroup.datasets['SALINITY'].changeColName('NONE', 'SALINITY')
if 'WINDSPEED' in ancGroup.datasets:
ancGroup.datasets['WINDSPEED'].changeColName('NONE', 'WINDSPEED')
if 'SST' in ancGroup.datasets:
ancGroup.datasets['SST'].changeColName('NONE', 'SST')
if satnavGroup:
ancGroup.datasets['REL_AZ'] = satnavGroup.getDataset('REL_AZ')
if 'HUMIDITY' in ancGroup.datasets:
ancGroup.datasets['HUMIDITY'] = satnavGroup.getDataset(
'HUMIDITY')
ancGroup.datasets['HUMIDITY'].changeColName('NONE', 'HUMIDITY')
# ancGroup.datasets['HEADING'] = satnavGroup.getDataset('HEADING') # Use GPS heading instead
ancGroup.addDataset('POINTING')
ancGroup.datasets['POINTING'] = satnavGroup.getDataset('POINTING')
ancGroup.datasets['POINTING'].changeColName('ROTATOR', 'POINTING')
ancGroup.datasets['REL_AZ'] = satnavGroup.getDataset('REL_AZ')
ancGroup.datasets['REL_AZ'].datasetToColumns()
# Use PITCH and ROLL preferentially from SolarTracker
if 'PITCH' in satnavGroup.datasets:
ancGroup.addDataset('PITCH')
ancGroup.datasets['PITCH'] = satnavGroup.getDataset('PITCH')
ancGroup.datasets['PITCH'].changeColName('SAS', 'PITCH')
if 'ROLL' in satnavGroup.datasets:
ancGroup.addDataset('ROLL')
ancGroup.datasets['ROLL'] = satnavGroup.getDataset('ROLL')
ancGroup.datasets['ROLL'].changeColName('SAS', 'ROLL')
if 'NONE' in ancGroup.datasets['REL_AZ'].columns:
ancGroup.datasets['REL_AZ'].changeColName('NONE', 'REL_AZ')
if pyrGroup is not None:
#PYROMETER
ancGroup.datasets['SST_IR'] = pyrGroup.getDataset("T")
ancGroup.datasets['SST_IR'].datasetToColumns()
ancGroup.datasets['SST_IR'].changeColName('IR', 'SST_IR')
# At this stage, all datasets in all groups of node have Timetag2
# and Datetag incorporated into data arrays. Calculate and add
# Datetime to each data array.
Utilities.rootAddDateTimeCol(node)
#################################################################################
# Filter the spectra from the entire collection before slicing the intervals at L2
##################################################################################
# Lt Quality Filtering; anomalous elevation in the NIR
if ConfigFile.settings["bL1bqcLtUVNIR"]:
msg = "Applying Lt(NIR)>Lt(UV) quality filtering to eliminate spectra."
print(msg)
Utilities.writeLogFile(msg)
# This is not well optimized for large files...
badTimes = ProcessL1bqc.ltQuality(sasGroup)
if badTimes is not None:
print('Removing records... Can be slow for large files')
check = Utilities.filterData(referenceGroup, badTimes)
# check is now fraction removed
# I.e., if >99% of the Es spectra from this entire file were remove, abort this file
if check > 0.99:
msg = "Too few spectra remaining. Abort."
print(msg)
Utilities.writeLogFile(msg)
return False
Utilities.filterData(sasGroup, badTimes)
Utilities.filterData(ancGroup, badTimes)
# Filter low SZAs and high winds after interpolating model/ancillary data
maxWind = float(ConfigFile.settings["fL1bqcMaxWind"])
wind = ancGroup.getDataset("WINDSPEED").data["WINDSPEED"]
timeStamp = ancGroup.datasets["WINDSPEED"].columns["Datetime"]
badTimes = None
i = 0
start = -1
stop = []
for index, _ in enumerate(wind):
if wind[index] > maxWind:
i += 1
if start == -1:
msg = f'High Wind: {round(wind[index])}'
Utilities.writeLogFile(msg)
start = index
stop = index
if badTimes is None:
badTimes = []
else:
if start != -1:
msg = f'Passed. Wind: {round(wind[index])}'
print(msg)
Utilities.writeLogFile(msg)
startstop = [timeStamp[start], timeStamp[stop]]
msg = f' Flag data from TT2: {startstop[0]} to {startstop[1]}'
# print(msg)
Utilities.writeLogFile(msg)
badTimes.append(startstop)
start = -1
end_index = index
msg = f'Percentage of data out of Wind limits: {round(100*i/len(timeStamp))} %'
print(msg)
Utilities.writeLogFile(msg)
if start != -1 and stop == end_index: # Records from a mid-point to the end are bad
startstop = [timeStamp[start], timeStamp[stop]]
msg = f' Flag data from TT2: {startstop[0]} to {startstop[1]}'
# print(msg)
Utilities.writeLogFile(msg)
if badTimes is None: # only one set of records
badTimes = [startstop]
else:
badTimes.append(startstop)
if start == 0 and stop == end_index: # All records are bad
return False
if badTimes is not None and len(badTimes) != 0:
print('Removing records...')
check = Utilities.filterData(referenceGroup, badTimes)
if check > 0.99:
msg = "Too few spectra remaining. Abort."
print(msg)
Utilities.writeLogFile(msg)
return False
Utilities.filterData(sasGroup, badTimes)
Utilities.filterData(ancGroup, badTimes)
# Filter SZAs
SZAMin = float(ConfigFile.settings["fL1bqcSZAMin"])
SZAMax = float(ConfigFile.settings["fL1bqcSZAMax"])
SZA = ancGroup.datasets["SZA"].columns["SZA"]
timeStamp = ancGroup.datasets["SZA"].columns["Datetime"]
badTimes = None
i = 0
start = -1
stop = []
for index, _ in enumerate(SZA):
if SZA[index] < SZAMin or SZA[index] > SZAMax or wind[
index] > maxWind:
i += 1
if start == -1:
msg = f'Low SZA. SZA: {round(SZA[index])}'
print(msg)
Utilities.writeLogFile(msg)
start = index
stop = index
if badTimes is None:
badTimes = []
else:
if start != -1:
msg = f'Passed. SZA: {round(SZA[index])}'
print(msg)
Utilities.writeLogFile(msg)
startstop = [timeStamp[start], timeStamp[stop]]
msg = f' Flag data from TT2: {startstop[0]} to {startstop[1]}'
# print(msg)
Utilities.writeLogFile(msg)
badTimes.append(startstop)
start = -1
end_index = index
msg = f'Percentage of data out of SZA limits: {round(100*i/len(timeStamp))} %'
print(msg)
Utilities.writeLogFile(msg)
if start != -1 and stop == end_index: # Records from a mid-point to the end are bad
startstop = [timeStamp[start], timeStamp[stop]]
msg = f' Flag data from TT2: {startstop[0]} to {startstop[1]}'
# print(msg)
Utilities.writeLogFile(msg)
if badTimes is None: # only one set of records
badTimes = [startstop]
else:
badTimes.append(startstop)
if start == 0 and stop == end_index: # All records are bad
return False
if badTimes is not None and len(badTimes) != 0:
print('Removing records...')
check = Utilities.filterData(referenceGroup, badTimes)
if check > 0.99:
msg = "Too few spectra remaining. Abort."
print(msg)
Utilities.writeLogFile(msg)
return False
Utilities.filterData(sasGroup, badTimes)
Utilities.filterData(ancGroup, badTimes)
# Spectral Outlier Filter
enableSpecQualityCheck = ConfigFile.settings[
'bL1bqcEnableSpecQualityCheck']
if enableSpecQualityCheck:
badTimes = None
msg = "Applying spectral filtering to eliminate noisy spectra."
print(msg)
Utilities.writeLogFile(msg)
inFilePath = node.attributes['In_Filepath']
badTimes1 = ProcessL1bqc.specQualityCheck(referenceGroup,
inFilePath)
badTimes2 = ProcessL1bqc.specQualityCheck(sasGroup, inFilePath)
if badTimes1 is not None and badTimes2 is not None:
badTimes = np.append(badTimes1, badTimes2, axis=0)
elif badTimes1 is not None:
badTimes = badTimes1
elif badTimes2 is not None:
badTimes = badTimes2
if badTimes is not None:
print('Removing records...')
check = Utilities.filterData(referenceGroup, badTimes)
if check > 0.99:
msg = "Too few spectra remaining. Abort."
print(msg)
Utilities.writeLogFile(msg)
return False
check = Utilities.filterData(sasGroup, badTimes)
if check > 0.99:
msg = "Too few spectra remaining. Abort."
print(msg)
Utilities.writeLogFile(msg)
return False
check = Utilities.filterData(ancGroup, badTimes)
if check > 0.99:
msg = "Too few spectra remaining. Abort."
print(msg)
Utilities.writeLogFile(msg)
return False
# Next apply the Meteorological Filter prior to slicing
esData = referenceGroup.getDataset("ES")
enableMetQualityCheck = int(
ConfigFile.settings["bL1bqcEnableQualityFlags"])
if enableMetQualityCheck:
msg = "Applying meteorological filtering to eliminate spectra."
print(msg)
Utilities.writeLogFile(msg)
badTimes = ProcessL1bqc.metQualityCheck(referenceGroup, sasGroup)
if badTimes is not None:
if len(badTimes) == esData.data.size:
msg = "All data flagged for deletion. Abort."
print(msg)
Utilities.writeLogFile(msg)
return False
print('Removing records...')
check = Utilities.filterData(referenceGroup, badTimes)
if check > 0.99:
msg = "Too few spectra remaining. Abort."
print(msg)
Utilities.writeLogFile(msg)
return False
Utilities.filterData(sasGroup, badTimes)
Utilities.filterData(ancGroup, badTimes)
return True
