def includeModelDefaults(ancGroup, modRoot):
''' Include model data or defaults for blank ancillary fields '''
print(
'Filling blank ancillary data with models or defaults from Configuration'
)
epoch = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)
# radData = referenceGroup.getDataset("ES") # From node, the input file
# Convert ancillary date time
if ancGroup is not None:
ancGroup.datasets['LATITUDE'].datasetToColumns()
ancTime = ancGroup.datasets['LATITUDE'].columns['Timetag2']
ancSeconds = []
ancDatetime = []
for i, ancDate in enumerate(
ancGroup.datasets['LATITUDE'].columns['Datetag']):
ancDatetime.append(
Utilities.timeTag2ToDateTime(
Utilities.dateTagToDateTime(ancDate), ancTime[i]))
ancSeconds.append((ancDatetime[i] - epoch).total_seconds())
# Convert model data date and time to datetime and then to seconds for interpolation
if modRoot is not None:
modTime = modRoot.groups[0].datasets["Timetag2"].tolist()
modSeconds = []
modDatetime = []
for i, modDate in enumerate(
modRoot.groups[0].datasets["Datetag"].tolist()):
modDatetime.append(
Utilities.timeTag2ToDateTime(
Utilities.dateTagToDateTime(modDate), modTime[i]))
modSeconds.append((modDatetime[i] - epoch).total_seconds())
# Model or default fills
if 'WINDSPEED' in ancGroup.datasets:
ancGroup.datasets['WINDSPEED'].datasetToColumns()
windDataset = ancGroup.datasets['WINDSPEED']
wind = windDataset.columns['NONE']
else:
windDataset = ancGroup.addDataset('WINDSPEED')
wind = np.empty((1, len(ancSeconds)))
wind[:] = np.nan
wind = wind[0].tolist()
if 'AOD' in ancGroup.datasets:
ancGroup.datasets['AOD'].datasetToColumns()
aodDataset = ancGroup.datasets['AOD']
aod = aodDataset.columns['NONE']
else:
aodDataset = ancGroup.addDataset('AOD')
aod = np.empty((1, len(ancSeconds)))
aod[:] = np.nan
aod = aod[0].tolist()
# Default fills
if 'SALINITY' in ancGroup.datasets:
ancGroup.datasets['SALINITY'].datasetToColumns()
saltDataset = ancGroup.datasets['SALINITY']
salt = saltDataset.columns['NONE']
else:
saltDataset = ancGroup.addDataset('SALINITY')
salt = np.empty((1, len(ancSeconds)))
salt[:] = np.nan
salt = salt[0].tolist()
if 'SST' in ancGroup.datasets:
ancGroup.datasets['SST'].datasetToColumns()
sstDataset = ancGroup.datasets['SST']
sst = sstDataset.columns['NONE']
else:
sstDataset = ancGroup.addDataset('SST')
sst = np.empty((1, len(ancSeconds)))
sst[:] = np.nan
sst = sst[0].tolist()
# Initialize flags
windFlag = []
aodFlag = []
for i, ancSec in enumerate(ancSeconds):
if np.isnan(wind[i]):
windFlag.append('undetermined')
else:
windFlag.append('field')
if np.isnan(aod[i]):
aodFlag.append('undetermined')
else:
aodFlag.append('field')
# Replace Wind, AOD NaNs with modeled data where possible.
# These will be within one hour of the field data.
if modRoot is not None:
msg = 'Filling in field data with model data where needed.'
print(msg)
Utilities.writeLogFile(msg)
for i, ancSec in enumerate(ancSeconds):
if np.isnan(wind[i]):
# msg = 'Replacing wind with model data'
# print(msg)
# Utilities.writeLogFile(msg)
idx = Utilities.find_nearest(modSeconds, ancSec)
wind[i] = modRoot.groups[0].datasets['Wind'][idx]
windFlag[i] = 'model'
if np.isnan(aod[i]):
# msg = 'Replacing AOD with model data'
# print(msg)
# Utilities.writeLogFile(msg)
idx = Utilities.find_nearest(modSeconds, ancSec)
aod[i] = modRoot.groups[0].datasets['AOD'][idx]
aodFlag[i] = 'model'
# Replace Wind, AOD, SST, and Sal with defaults where still nan
msg = 'Filling in ancillary data with default values where still needed.'
print(msg)
Utilities.writeLogFile(msg)
saltFlag = []
sstFlag = []
for i, value in enumerate(wind):
if np.isnan(value):
wind[i] = ConfigFile.settings["fL1bqcDefaultWindSpeed"]
windFlag[i] = 'default'
for i, value in enumerate(aod):
if np.isnan(value):
aod[i] = ConfigFile.settings["fL1bqcDefaultAOD"]
aodFlag[i] = 'default'
for i, value in enumerate(salt):
if np.isnan(value):
salt[i] = ConfigFile.settings["fL1bqcDefaultSalt"]
saltFlag.append('default')
else:
saltFlag.append('field')
for i, value in enumerate(sst):
if np.isnan(value):
sst[i] = ConfigFile.settings["fL1bqcDefaultSST"]
sstFlag.append('default')
else:
sstFlag.append('field')
# Populate the datasets and flags with the InRad variables
windDataset.columns["NONE"] = wind
windDataset.columns["WINDFLAG"] = windFlag
windDataset.columnsToDataset()
aodDataset.columns["AOD"] = aod
aodDataset.columns["AODFLAG"] = aodFlag
aodDataset.columnsToDataset()
saltDataset.columns["NONE"] = salt
saltDataset.columns["SALTFLAG"] = saltFlag
saltDataset.columnsToDataset()
sstDataset.columns["NONE"] = sst
sstDataset.columns["SSTFLAG"] = sstFlag
sstDataset.columnsToDataset()
# Convert ancillary seconds back to date/timetags ...
ancDateTag = []
ancTimeTag2 = []
ancDT = []
for i, sec in enumerate(ancSeconds):
ancDT.append(
datetime.datetime.utcfromtimestamp(sec).replace(
tzinfo=datetime.timezone.utc))
ancDateTag.append(
float(
f'{int(ancDT[i].timetuple()[0]):04}{int(ancDT[i].timetuple()[7]):03}'
))
ancTimeTag2.append(float( \
f'{int(ancDT[i].timetuple()[3]):02}{int(ancDT[i].timetuple()[4]):02}{int(ancDT[i].timetuple()[5]):02}{int(ancDT[i].microsecond/1000):03}'))
# Move the Timetag2 and Datetag into the arrays and remove the datasets
for ds in ancGroup.datasets:
ancGroup.datasets[ds].columns["Datetag"] = ancDateTag
ancGroup.datasets[ds].columns["Timetag2"] = ancTimeTag2
ancGroup.datasets[ds].columns["Datetime"] = ancDT
ancGroup.datasets[ds].columns.move_to_end('Timetag2', last=False)
ancGroup.datasets[ds].columns.move_to_end('Datetag', last=False)
ancGroup.datasets[ds].columns.move_to_end('Datetime', last=False)
ancGroup.datasets[ds].columnsToDataset()
def getAnc(inputGroup):
''' Retrieve model data and save in Data/Anc and in ModData '''
server = 'oceandata.sci.gsfc.nasa.gov'
cwd = os.getcwd()
if not os.path.exists(os.path.join(cwd, "Data", "Anc")):
os.makedirs(os.path.join(cwd, "Data", "Anc"))
# Get the dates, times, and locations from the input group
latDate = inputGroup.getDataset('LATITUDE').data["Datetag"]
latTime = inputGroup.getDataset('LATITUDE').data["Timetag2"]
lat = inputGroup.getDataset('LATITUDE').data["NONE"]
lon = inputGroup.getDataset('LONGITUDE').data["NONE"]
modWind = []
modAOD = []
# Loop through the input group and extract model data for each element
oldFile = None
for index, dateTag in enumerate(latDate):
dateTagNew = Utilities.dateTagToDate(dateTag)
year = int(str(int(dateTagNew))[0:4])
month = int(str(int(dateTagNew))[4:6])
day = int(str(int(dateTagNew))[6:8])
# doy = int(str(int(dateTag))[4:7])
# Casting below can push hr to 24. Truncate the hr decimal using
# int() so the script always calls from within the hour in question,
# and no rounding occurs.
hr = int(Utilities.timeTag2ToSec(latTime[index]) / 60 / 60)
file1 = f"GMAO_MERRA2.{year}{month:02.0f}{day:02.0f}T{hr:02.0f}0000.MET.nc"
if oldFile != file1:
ancPath = os.path.join(cwd, "Data", "Anc")
filePath1 = os.path.join(cwd, "Data", "Anc", file1)
if not os.path.exists(filePath1):
request = f"/cgi/getfile/{file1}"
msg = f'Retrieving anchillary file from server: {file1}'
print(msg)
Utilities.writeLogFile(msg)
status = OBPGSession.httpdl(server,
request,
localpath=ancPath,
outputfilename=file1,
uncompress=False,
verbose=2)
else:
status = 200
msg = f'Ancillary file found locally: {file1}'
print(msg)
Utilities.writeLogFile(msg)
file2 = f"GMAO_MERRA2.{year}{month:02.0f}{day:02.0f}T{hr:02.0f}0000.AER.nc"
filePath2 = os.path.join(cwd, "Data", "Anc", file2)
if not os.path.exists(filePath2):
request = f"/cgi/getfile/{file2}"
msg = f'Retrieving anchillary file from server: {file2}'
print(msg)
Utilities.writeLogFile(msg)
status = OBPGSession.httpdl(server,
request,
localpath=ancPath,
outputfilename=file2,
uncompress=False,
verbose=2)
else:
status = 200
msg = f'Ancillary file found locally: {file2}'
print(msg)
Utilities.writeLogFile(msg)
if status in (400, 401, 403, 404, 416):
msg = f'Request error: {status}'
print(msg)
Utilities.writeLogFile(msg)
alert = QtWidgets.QMessageBox()
alert.setText(f'Request error: {status}\n \
Enter server credentials in the\n \
Configuration Window L1BQC')
alert.exec_()
return None
# GMAO Atmospheric model data
node = HDFRoot.readHDF5(filePath1)
root = HDFRoot()
root.copyAttributes(node)
# dataset are read into root level
gmaoGroup = root.addGroup('GMAO')
for ds in node.datasets:
name = ds.id
newds = gmaoGroup.addDataset(name)
newds.columns["None"] = ds.data[:].tolist()
newds.columnsToDataset()
# extract and return ancillary data from netcdf4 files....
ancLat = np.array(gmaoGroup.getDataset("lat").data.tolist())
ancLon = np.array(gmaoGroup.getDataset("lon").data.tolist())
# Humidity
# not needed
# Wind
ancUwind = gmaoGroup.getDataset(
"U10M") # Eastward at 10m [m/s]
ancVwind = gmaoGroup.getDataset("V10M") # Northward
# Aerosols
node = HDFRoot.readHDF5(filePath2)
root = HDFRoot()
root.copyAttributes(node)
# dataset are read into root level
aerGroup = root.addGroup('AEROSOLS')
for ds in node.datasets:
name = ds.id
newds = aerGroup.addDataset(name)
newds.columns["None"] = ds.data[:].tolist()
newds.columnsToDataset()
# extract and return ancillary data from netcdf4 files....
ancLatAer = np.array(aerGroup.getDataset("lat").data.tolist())
ancLonAer = np.array(aerGroup.getDataset("lon").data.tolist())
# Total Aerosol Extinction AOT 550 nm, same as AOD(550)
ancTExt = aerGroup.getDataset("TOTEXTTAU")
oldFile = file1
# Locate the relevant cell
latInd = Utilities.find_nearest(ancLat, lat[index])
lonInd = Utilities.find_nearest(ancLon, lon[index])
# position retrieval index has been confirmed manually in SeaDAS
uWind = ancUwind.data["None"][latInd][lonInd]
vWind = ancVwind.data["None"][latInd][lonInd]
modWind.append(np.sqrt(uWind * uWind +
vWind * vWind)) # direction not needed
# Locate the relevant cell
latInd = Utilities.find_nearest(ancLatAer, lat[index])
lonInd = Utilities.find_nearest(ancLonAer, lon[index])
# position confirmed in SeaDAS
modAOD.append(ancTExt.data["None"][latInd][lonInd])
modData = HDFRoot()
modGroup = modData.addGroup('MERRA2_model')
modGroup.addDataset('Datetag')
modGroup.addDataset('Timetag2')
modGroup.addDataset('AOD')
modGroup.addDataset('Wind')
modGroup.datasets['Datetag'] = latDate
modGroup.datasets['Timetag2'] = latTime
modGroup.datasets['AOD'] = modAOD
modGroup.datasets['Wind'] = modWind
print('GetAnc: Model data retrieved')
return modData