def check_uvis_limbs(regex, file_level):
"""plot and print uvis limbs with minimum tangent height and spectral binning data"""
hdf5_files, hdf5_filenames, _ = make_filelist(regex, file_level)
colours = get_colours(len(hdf5_filenames))
fig, ax = plt.subplots()
for file_index, (hdf5_file, hdf5_filename) in enumerate(
zip(hdf5_files, hdf5_filenames)):
tangent_alts = np.mean(
hdf5_file["Geometry/Point0/TangentAltAreoid"][...], axis=1)
valid_indices = np.where(tangent_alts > -990.0)[0]
ax.plot(tangent_alts[valid_indices],
label=hdf5_filename,
color=colours[file_index])
binning = hdf5_file["Channel/HorizontalAndCombinedBinningSize"][0]
n_spectra = len(tangent_alts)
centre_indices = np.arange(int(n_spectra / 4), int(n_spectra * 3 / 4))
min_tangent_alt = np.min(tangent_alts[centre_indices])
print(hdf5_filename, "min alt=%0.1f" % min_tangent_alt,
"binning=%i" % binning)
ax.legend()
def getNumberOfSpectra(obspaths, fileLevel):
"""print number of spectra in each file"""
hdf5Files, hdf5Filenames, titles = make_filelist(obspaths, fileLevel)
for hdf5File, hdf5Filename in zip(hdf5Files, hdf5Filenames):
detectorData = hdf5File["Science/Y"][...]
nSpectra = detectorData.shape[0]
# if nSpectra == 920:
print("nSpectra=%i, hdf5Filename=%s" %(nSpectra, hdf5Filename))
#4 khz
# 20181010_084333_0p2a_SO_2_C
# 20190416_024455_0p2a_SO_1_C
#8 khz
# 20190107_015635_0p2a_SO_2_C
# 20190307_011600_0p2a_SO_1_C
#
regex = re.compile("20190223_054340_0p2a_SO_1_C")
# regex = re.compile("(20181010_084333_0p2a_SO_2_C|20190416_024455_0p2a_SO_1_C|20190107_015635_0p2a_SO_2_C|20190307_011600_0p2a_SO_1_C)")
SIMULATION_ADJACENT_ORDERS = 4
hdf5Files, hdf5Filenames, _ = make_filelist(regex, file_level)
linestyles = {0: "-", 1: ":", 2: "--", 3: "-."}
pixels = np.arange(320)
ss_file = os.path.join(paths["RETRIEVALS"]["SOLAR_DIR"],
"Solar_irradiance_ACESOLSPEC_2015.dat")
for hdf5_file, hdf5_filename in zip(hdf5Files, hdf5Filenames):
print(hdf5_filename)
channel = hdf5_filename.split("_")[3].lower()
# detector_rows = {"so":[128-8, 128+8], "lno":[152-72, 152+72]}[channel]
d = {
"text": [],
# load_spice_kernels()
#SAVE_FIGS = False
SAVE_FIGS = True
# SAVE_FILES = False
# #SAVE_FILES = True
file_level = "hdf5_level_0p3k"
# regex = re.compile("20(18|19|20)[0-9][0-9][0-9][0-9]_.*_0p3k_SO_A_[IE]_134")
regex = re.compile("20(18|19)[0-9][0-9][0-9][0-9]_.*_0p3k_SO_A_[IE]_(134|136)")
# regex = re.compile("20180[456][0-9][0-9]_.*_0p3k_SO_A_[IE]_(134|136)")
#get files
hdf5_files, hdf5_filenames, titles = make_filelist(regex,
file_level,
open_files=False)
#loop through files
obs_datetimes = []
relative_signals = []
for file_index, (hdf5_file,
hdf5_filename) in enumerate(zip(hdf5_files, hdf5_filenames)):
if np.mod(file_index, 100) == 0:
print(file_index, hdf5_filename)
year = hdf5_filename[0:4]
month = hdf5_filename[4:6]
day = hdf5_filename[6:8]
regex = re.compile(".*_SO_._S") #row120 used
file_level = "hdf5_level_1p0a"
search_diffraction_order = 152
# vertical_line_nu = 0.0
vertical_line_nu = 3423.93 #add a line on the plots in a given position
y_limits = [0.05, 1.01]
with PdfPages("order_%s_fullscans.pdf" %
search_diffraction_order) as pdf: #open pdf
hdf5_files, hdf5_filenames, _ = make_filelist(regex,
file_level,
silent=True)
for hdf5_file, hdf5_filename in zip(hdf5_files, hdf5_filenames):
print(hdf5_filename)
fig, ax = plt.subplots(figsize=(12, 9))
diffraction_orders = hdf5_file["Channel/DiffractionOrder"][...]
alts = hdf5_file["Geometry/Point0/TangentAltAreoid"][:, 0]
indices = [
i for i, x in enumerate(diffraction_orders)
if x == search_diffraction_order
]
"_position_of_illumination_on_detector.png")
plt.figure(figsize=(FIG_X, FIG_Y))
plt.title(titles[fileIndex] +
": illumination intensity per detector row")
plt.xlabel("Detector row")
plt.ylabel("Detector counts (bg subtracted)")
for index in range(0, np.int(nSpectra / 24)):
plt.plot(bins[range(24 * index, 24 * (index + 1)), 0],
detectorData[range(24 * index, 24 * (index + 1)), 160])
# plt.plot(bins[range(24*index,24*(index+1)),0],normalise(detectorData[range(24*index,24*(index+1)),160]))
if SAVE_FIGS:
plt.savefig(titles[fileIndex].replace(" ", "_") +
"_illumination_intensity_per_detector_row.png")
return detectorFrame
"""plot linescans"""
hdf5Files, hdf5Filenames, titles = make_filelist(obspaths, fileLevel)
analyseLineScan2(hdf5Files, hdf5Filenames)
#ani = animateFrames(hdf5Files, hdf5Filenames, titles); plt.show(plt.show())
#don't trust the calculation - angles estimated manually from data
#angleSeparationA = -89.08792599580298
#angleSeparationB = 67.15504926474100 + 3.0/60.0 #old value plus 3 arcminutes
#printBoresights(angleSeparationA, angleSeparationB)
"""plot full detector frame"""
#hdf5Files, hdf5Filenames, titles = makeFileList(obspaths, fileLevel)
#detectorFrame = showFullFrame03A(hdf5Files, hdf5Filenames, titles, frameNumbers)
order_data_dict = {}
for diffraction_order, order_dict in order_dicts.items():
order_data_dict[diffraction_order] = {}
order_data_dict[diffraction_order]["mean_gradient_all_bins"] = []
order_data_dict[diffraction_order]["std_gradient_all_bins"] = []
order_data_dict[diffraction_order]["n_gradients_all_bins"] = []
if DATA_TYPE == "ground":
order_data_dict[diffraction_order]["obspaths_all"] = getFilenameList("ground cal %s cell%s" %(order_dict["molecule"], SUFFIX))
hdf5Files, hdf5Filenames, _ = make_filelist(order_data_dict[diffraction_order]["obspaths_all"], fileLevel, model=model, silent=True)
elif DATA_TYPE == "inflight":
regex = re.compile("(20161121_233000|20180702_112352|20181101_213226|20190314_021825|20190609_011514|20191207_051654)_0p1a_LNO_1")
hdf5Files, hdf5Filenames, _ = make_filelist(regex, fileLevel)
order_data_dict["hdf5Filenames"] = {}
order_data_dict[diffraction_order]["measurement_temperatures"] = []
order_data_dict[diffraction_order]["colour"] = []
order_data_dict[diffraction_order]["hdf5_filenames"] = []
order_data_dict[diffraction_order]["spectra"] = []
order_data_dict[diffraction_order]["continuum_mean"] = []
order_data_dict[diffraction_order]["continuum_std"] = []
MAKE_AUX_FILE = True
# MAKE_AUX_FILE = False
CLOSE_FIGS = True
# CLOSE_FIGS = False
# TEMPERATURE_GRID_TO_USE = "measurement_temperature"
TEMPERATURE_GRID_TO_USE = "calculated_temperature"
# TEMPERATURE_GRID_TO_USE = "mean_temperature"
# regex = re.compile("(20161121_233000|20180702_112352|20181101_213226|20190314_021825|20190609_011514|20191207_051654)_0p1a_LNO_1")
regex = re.compile(
"(20161121_233000|20180702_112352|20181101_213226|20190314_021825|20190609_011514|20190921_222413|20191207_051654|20200105_132318|20200324_145739)_0p1a_LNO_1"
)
fileLevel = "hdf5_level_0p1a"
hdf5Files, hdf5Filenames, titles = make_filelist(regex, fileLevel)
"""plot solar lines in solar fullscan data for orders contains strong solar lines"""
temperature_range = np.arange(-20., 15., 0.1)
pixels = np.arange(320.0)
colours = get_colours(len(temperature_range), "plasma")
output_title = "LNO_Reflectance_Factor_Calibration_Table"
if MAKE_AUX_FILE:
hdf5_file_out = h5py.File(
os.path.join(paths["BASE_DIRECTORY"], output_title + ".h5"), "w")
#for diffraction_order in [116]:
for diffraction_order in ref_fact_orders_dict.keys():
error = False
ax.legend()
else:
if len(temperatures) > 1:
coeffs = np.polyfit(temperatures, values, 1)
fit = np.polyval(coeffs, temperatures)
rms = rmse(fit, values)
return coeffs[0], rms
if title == "MCC line scan":
"""make vertical detector plots where sun is seen to determine slit position and time when in centre"""
DETECTOR_V_CENTRE = 201
DETECTOR_CUTOFF = 20000
hdf5Files, hdf5Filenames, _ = make_filelist(obspaths,
fileLevel,
model=model)
for hdf5_file, hdf5_filename in zip(hdf5Files, hdf5Filenames):
channel = hdf5_filename.split("_")[3].lower()
cmap = plt.get_cmap('Set1')
window_data, unique_window_tops, detector_row_numbers = splitWindowSteppingByWindow(
hdf5_file)
nColours = 0
illuminated_window_tops = []
for window_frames, unique_window_top in zip(window_data,
unique_window_tops):
# print(np.max(window_frames[:, :, DETECTOR_V_CENTRE]))
if np.max(window_frames[:, :,
def plotDiffractionOrderBarChart(regex, fileLevel, channel):
hdf5Files, hdf5Filenames, titles = make_filelist(regex, fileLevel, open_files=False, silent=True)
if channel in ["so", "lno"]:
orders = range(200)
diffractionOrderFilenames = [[] for _ in orders]
for hdf5_filename in hdf5Filenames:
hdf5_filename_split = hdf5_filename.split("_")
if len(hdf5_filename_split) == 7:
diffractionOrder = hdf5_filename.split("_")[6]
diffractionOrderFilenames[int(diffractionOrder)].append(hdf5_filename)
# else:
# if hdf5_filename_split[-1] == "S":
# name, hdf5_file = get_file(hdf5_filename, fileLevel, 0, silent=True)
# detectorData = hdf5_file["Science/Y"][...]
# nSpectra = detectorData.shape[0]
# print("nSpectra=%i, hdf5_filename=%s" %(nSpectra, hdf5_filename))
# print(list(set(list(hdf5_file["Channel/DiffractionOrder"][...]))))
nDiffractionOrders = [len(values) for values in diffractionOrderFilenames]
plt.figure(figsize=(FIG_X, FIG_Y))
plt.bar(orders[118:198], nDiffractionOrders[118:198])
plt.title("Number of times each diffraction order was measured for search %s" %regex.pattern)
plt.ylabel("Number of observations")
plt.xlabel("Diffraction Order")
if SAVE_FIGS:
plt.savefig(os.path.join(paths["BASE_DIRECTORY"], "%s_diffraction_order_statistics.png" %channel.lower()))
elif channel in ["uvis"]:
uvisObsTypes = {
"Occultation\nfull spectrum\nfull resolution":0, \
"Occultation\nfull spectrum\nreduced resolution":1, \
"Occultation\nreduced spectrum\nfull resolution":2, \
"Nadir\nfull spectrum\nfull resolution":3, \
"Nadir\nfull spectrum\nreduced resolution":4, \
"Nadir\nreduced spectrum\nfull resolution":5, \
}
uvisObsTypeFilenames = [[] for _ in list(uvisObsTypes.keys())]
for file_index, hdf5_filename in enumerate(hdf5Filenames):
if len(hdf5Filenames) > 100:
if np.mod(file_index, 100) == 0:
print("Processing %s files %i/%i" %(fileLevel, file_index, len(hdf5Filenames)))
hdf5_filename_split = hdf5_filename.split("_")
if len(hdf5_filename_split) == 5:
observationType = hdf5_filename.split("_")[4]
if observationType in ["I", "E"]:
name, hdf5_file = get_file(hdf5_filename, fileLevel, 0, silent=True)
h_end = hdf5_file["Channel/HEnd"][0]
if h_end == 1047:
binning = hdf5_file["Channel/HorizontalAndCombinedBinningSize"][0]
if binning == 0:
index = uvisObsTypes["Occultation\nfull spectrum\nfull resolution"]
else:
index = uvisObsTypes["Occultation\nfull spectrum\nreduced resolution"]
elif h_end < 1047:
index = uvisObsTypes["Occultation\nreduced spectrum\nfull resolution"]
elif observationType in ["D"]:
name, hdf5_file = get_file(hdf5_filename, fileLevel, 0, silent=True)
h_end = hdf5_file["Channel/HEnd"][0]
if h_end == 1047:
binning = hdf5_file["Channel/HorizontalAndCombinedBinningSize"][0]
if binning == 0:
index = uvisObsTypes["Nadir\nfull spectrum\nfull resolution"]
else:
index = uvisObsTypes["Nadir\nfull spectrum\nreduced resolution"]
elif h_end < 1047:
index = uvisObsTypes["Nadir\nreduced spectrum\nfull resolution"]
uvisObsTypeFilenames[index].append(hdf5_filename)
nObsTypes = [len(values) for values in uvisObsTypeFilenames]
plt.figure(figsize=(FIG_X+1, FIG_Y+1))
plt.bar(range(len(uvisObsTypes)), nObsTypes)
plt.title("Number of times each observation type was measured for search %s" %regex.pattern)
plt.ylabel("Number of observations")
plt.xlabel("Observation Type")
plt.xticks(range(len(uvisObsTypes)), list(uvisObsTypes.keys()))
plt.tight_layout()
if SAVE_FIGS:
plt.savefig(os.path.join(BASE_DIRECTORY, "%s_observation_type_statistics.png" %channel.lower()))
def plotDarkCurrentResidual(regex, fileLevel, diffractionOrder):
#if True:
diffractionOrder = 171
chosenAotfFrequency = {119:15657, 132:17566, 133:17712, 134:17859, 135:18005, 136:18152, 149:20052, \
167:22674, 168:22820, 169:22965, 170:23110, 171:23255, 189:25864, 190:26008, 191:26153}[diffractionOrder]
hdf5Files, hdf5Filenames, titles = make_filelist(regex, fileLevel)
fig, ax1 = plt.subplots()
ax2 = ax1.twinx()
for fileIndex, (hdf5_file, hdf5_filename) in enumerate(zip(hdf5Files, hdf5Filenames)):
print("%i/%i: Reading in file %s" %(fileIndex, len(hdf5Filenames), hdf5_filename))
aotfFrequency = hdf5_file["Channel/AOTFFrequency"][...]
#plot 1x dark frames only
# if aotfFrequency[0] in singleAotfFrequencies and aotfFrequency[0] == aotfFrequency[1] == aotfFrequency[2] == aotfFrequency[3]:
#plot 5x dark frames only
# if aotfFrequency[0] in singleAotfFrequencies and aotfFrequency[1] == aotfFrequency[2] == aotfFrequency[3] == aotfFrequency[4] == 0.0:
if True:
print("%s, #%i" %(hdf5_filename, diffractionOrder))
detectorData = hdf5_file["Science/Y"][...]
lightIndices = np.where(aotfFrequency == chosenAotfFrequency)[0]
darkIndices = np.where(aotfFrequency == 0.0)[0]
detectorDataLight = np.asfarray([detectorData[index,:,:] for index in lightIndices])
detectorDataDark = np.asfarray([detectorData[index,:,:] for index in darkIndices])
chosenPixel = 200
for chosenBin in [0, 1, 2, 3]:
ax1.scatter(darkIndices, detectorDataDark[:, chosenBin, chosenPixel], s=5, marker="*", label="%s dark bin %i" %(hdf5_filename, chosenBin))
ax2.plot(lightIndices, detectorDataLight[:, chosenBin, chosenPixel], label="%s light bin %i" %(hdf5_filename, chosenBin))
lines, labels = ax1.get_legend_handles_labels()
lines2, labels2 = ax2.get_legend_handles_labels()
ax2.legend(lines + lines2, labels + labels2, loc=0)
ax2.set_xlabel("Frame index")
ax1.set_ylabel("Dark Frame Counts, Pixel %i" %(chosenPixel))
ax2.set_ylabel("Light Frame Counts, Pixel %i" %(chosenPixel))
if hdf5_filename == "20181223_045314_0p1a_SO_1":
sun_light_index = 1000 #light frame above atmosphere
sun_dark_index = 200 #dark frame above atmosphere
atmos_light_index = 1110 #light frame in atmosphere
atmos_dark_index = 222 #dark frame in atmosphere
mars_dark_index = 240 #real dark frame viewing mars
elif hdf5_filename == "20190116_182249_0p1a_SO_1":
sun_dark_index = 2338
mars_dark_index = 3666
elif hdf5_filename == "20181223_084859_0p1a_SO_1":
sun_light_index = 175 #light frame above atmosphere
sun_dark_index = 175 #dark frame above atmosphere
if diffractionOrder == 171:
atmos_light_index = 218 #light frame in atmosphere
atmos_dark_index = 218 #dark frame in atmosphere
if diffractionOrder == 191:
atmos_light_index = 216 #light frame in atmosphere
atmos_dark_index = 216 #dark frame in atmosphere
mars_dark_index = 225 #real dark frame viewing mars
else:
stop()
#generate pixel bin map
subtractedResidualMap = np.asfarray([detectorDataDark[sun_dark_index, index, :] - detectorDataDark[mars_dark_index, index, :] for index in range(4)])
plt.figure()
plt.imshow(subtractedResidualMap, aspect=20) #where 300000 is the light frame counts
plt.colorbar()
plt.xlabel("Pixel Number")
plt.ylabel("Bin Number")
plt.title("Signal difference between solar dark frame and mars dark frame")
plt.figure()
for chosenBin in [0, 1, 2, 3]:
plt.plot(detectorDataDark[sun_dark_index, chosenBin, :] - detectorDataDark[mars_dark_index, chosenBin, :], label="Bin %i" %chosenBin)
plt.legend()
plt.xlabel("Pixel Number")
plt.ylabel("Signal difference")
plt.title("Signal difference between solar dark frame and mars dark frame")
#see effect on transmittance using contaminated dark and mars dark
chosenBin = 3
sun_spectrum_straylight = detectorDataLight[sun_light_index, chosenBin, :] - detectorDataDark[sun_dark_index, chosenBin, :]
sun_spectrum = detectorDataLight[sun_light_index, chosenBin, :] - detectorDataDark[mars_dark_index, chosenBin, :]
atmos_spectrum_straylight = detectorDataLight[atmos_light_index, chosenBin, :] - detectorDataDark[atmos_dark_index, chosenBin, :]
atmos_spectrum = detectorDataLight[atmos_light_index, chosenBin, :] - detectorDataDark[mars_dark_index, chosenBin, :]
transmittance_straylight = atmos_spectrum_straylight / sun_spectrum_straylight
transmittance = atmos_spectrum / sun_spectrum
plt.figure()
plt.plot(transmittance_straylight, label="Normal transmittance calculation")
plt.plot(transmittance, label="Transmittance using true darks")
plt.legend()
plt.xlabel("Pixel Number")
plt.ylabel("Transmittance")
plt.title("Transmittance calculation comparison, \n%s order %i" %(hdf5_filename, diffractionOrder))
"""write list of Ls to file"""
#writeTimeLsToFile()
"""compare spectral coefficients and temperature dependent shifts"""
#pixelSpectralCoefficients = getDatasetFromFiles(hdf5Files, hdf5Filenames, "Channel/PixelSpectralCoefficients", first_value_only=True)
#pixel1 = getDatasetFromFiles(hdf5Files, hdf5Filenames, "Channel/Pixel1", first_value_only=True)
#temperature = getDatasetFromFiles(hdf5Files, hdf5Filenames, "Housekeeping/SENSOR_2_TEMPERATURE_LNO")
#xAll = getDatasetFromFiles(hdf5Files, hdf5Filenames, "Science/X", first_value_only=True)
#x0 = [value[0] for value in xAll]
#x319 = [value[319] for value in xAll]
"""plot relative signal strengths of SO channel bins to check alignment"""
hdf5Files, hdf5Filenames, titles = make_filelist(obspaths, fileLevel)
plotBinStrengths(hdf5Files, hdf5Filenames, obspaths)
"""plot dark current for 5 x darks and residual signal on detector bins"""
#if title == "plot dark residual":
# plotDarkCurrentResidual(regex, fileLevel, 149)
if title == "so order statistics":
channel = "so"
plotDiffractionOrderBarChart(regex, fileLevel, channel)
def process_channel_data(self, args):
"""make database containing info about all spectra in a channel for a particular observation type"""
self.level = args.level
self.command = args.command
if args.silent:
silent = True
else:
silent = False
table_fields = obs_database_fields(self.level,
bira_server=self.bira_server)
table_name = self.level
if args.regenerate:
print("Deleting and regenerating table")
self.check_if_table_exists(table_name)
self.drop_table(table_name)
self.new_table(table_name, table_fields)
print("Getting file list")
if args.regex:
regex = re.compile(args.regex)
else:
if self.command == "lno_nadir":
if self.level == "hdf5_level_1p0a":
regex = re.compile("20.*_LNO.*_D(P|F).*")
else:
regex = re.compile("20.*_LNO.*_D.*")
elif self.command == "so_occultation":
regex = re.compile("20.*_SO.*_[IE].*")
elif self.command == "uvis_nadir":
regex = re.compile("20.*_UVIS.*_D")
elif self.command == "uvis_occultation":
regex = re.compile("20.*_UVIS.*_[IE]")
beg_datetime = datetime.datetime.strptime(args.beg, ARG_FORMAT)
end_datetime = datetime.datetime.strptime(args.end, ARG_FORMAT)
_, hdf5Filenames, _ = make_filelist(regex,
self.level,
silent=silent,
open_files=False)
print("%i files found in directory" % len(hdf5Filenames))
#make datetime from hdf5 filenames, find those that match the beg/end times
hdf5_datetimes = [
datetime.datetime.strptime(i[:15], HDF5_FILENAME_FORMAT)
for i in hdf5Filenames
]
# hdf5_file_indices = [i for i, hdf5_datetime in enumerate(hdf5_datetimes) if beg_datetime < hdf5_datetime < end_datetime]
matching_hdf5_filenames = [
hdf5_filename for hdf5_datetime, hdf5_filename in zip(
hdf5_datetimes, hdf5Filenames)
if beg_datetime < hdf5_datetime < end_datetime
]
print("Adding %i files between %s and %s to database" %
(len(matching_hdf5_filenames), args.beg, args.end))
for fileIndex, hdf5Filename in enumerate(matching_hdf5_filenames):
hdf5Filepath = get_filepath(hdf5Filename)
if not silent:
print("Collecting data: file %i/%i: %s" %
(fileIndex, len(matching_hdf5_filenames), hdf5Filename))
with h5py.File(hdf5Filepath, "r") as hdf5File:
orbit = hdf5File.attrs["Orbit"]
filename = hdf5Filename
diffraction_order = hdf5File["Channel/DiffractionOrder"][0]
sbsf = hdf5File["Channel/BackgroundSubtraction"][0]
utc_start_times = hdf5File["Geometry/ObservationDateTime"][:,
0]
duration = get_obs_duration(hdf5File)
n_spectra = len(utc_start_times)
n_orders = hdf5File.attrs["NSubdomains"]
longitudes = hdf5File["Geometry/Point0/Lon"][:, 0]
latitudes = hdf5File["Geometry/Point0/Lat"][:, 0]
if self.command == "lno_nadir":
mean_temperature_tgo = np.mean(
hdf5File["Temperature/NominalLNO"][...])
bin_index = np.ones(n_spectra)
incidence_angles = hdf5File[
"Geometry/Point0/IncidenceAngle"][:, 0]
altitudes = np.zeros(n_spectra) - 999.0
elif self.command == "so_occultation":
mean_temperature_tgo = np.mean(
hdf5File["Temperature/NominalSO"][...])
bin_index = hdf5File["Channel/IndBin"][...]
incidence_angles = np.zeros(n_spectra) - 999.0
altitudes = hdf5File["Geometry/Point0/TangentAltAreoid"][:,
0]
local_times = hdf5File["Geometry/Point0/LST"][:, 0]
sql_table_rows = []
#get mean of y radiance factor continuum
if self.level == "hdf5_level_1p0a":
y = hdf5File["Science/YReflectanceFactor"][:, :]
for i in range(n_spectra):
if incidence_angles[i] < 80.0:
continuum = baseline_als(y[i, :])
y_mean = np.mean(continuum[160:240])
sql_table_rows.append([None, orbit, filename, i, mean_temperature_tgo, \
int(diffraction_order), int(sbsf), int(bin_index[i]), utc_start_times[i].decode(), \
duration, int(n_spectra), int(n_orders), longitudes[i], latitudes[i], \
altitudes[i], incidence_angles[i], local_times[i], float(y_mean)])
else:
for i in range(n_spectra):
sql_table_rows.append([None, orbit, filename, i, mean_temperature_tgo, \
int(diffraction_order), int(sbsf), int(bin_index[i]), utc_start_times[i].decode(), \
duration, int(n_spectra), int(n_orders), longitudes[i], latitudes[i], \
altitudes[i], incidence_angles[i], local_times[i]])
sql_table_rows_datetime = self.convert_table_datetimes(
table_fields, sql_table_rows)
# self.insert_rows(table_name, table_fields, sql_table_rows_datetime, check_duplicates=False)
self.insert_rows(table_name,
table_fields,
sql_table_rows_datetime,
check_duplicates=True)
from tools.general.length import length
HDF5_DT_FORMAT = "%Y %b %d %H:%M:%S.%f"
# year = "2020[0-9][0-9][0-9][0-9]"
regex = re.compile("20[0-9][0-9][0-9][0-9][0-9][0-9]_.*_SO_A_I_190")
#regex = re.compile("202005[0-9][0-9]_.*_SO_A_I_190")
file_level = "hdf5_level_0p3k"
datetime_strings = []
max_alts = []
lats = []
hdf5_filepaths, hdf5_filenames, _ = make_filelist(regex,
file_level,
open_files=False)
for hdf5_filepath, hdf5_filename in zip(hdf5_filepaths, hdf5_filenames):
with h5py.File(hdf5_filepath, "r") as f:
alts = f["Geometry/Point0/TangentAlt"][
121:123, 0] #30 seconds x 4 bins + 1 for bin 1
latitudes_in = f["Geometry/Point0/Lat"][:, 0]
datetime_in = f["Temperature/TemperatureDateTime"][...]
if length(datetime_in) > 20:
max_alts.append(np.mean(alts))
mid_point = int(len(datetime_in) / 2)
