def handle_common_config(template_obj, out_config_filename, used_files, sounding_ids, run_type, ids_file_keyword='L1BFile', id_list_sect=None): inp_prod_section = template_obj.get_section(INP_PROD_SECTION_NAME) if len(inp_prod_section) == 0: print(template_obj.get_all_section_names()) raise IOError('Could not find input product file section of %s' % template_obj.filename) for curr_file in used_files: keyword_name = check_file_type(curr_file) if keyword_name != None: inp_prod_section[0].set_keyword_value(keyword_name, curr_file) ids_file = inp_prod_section[0].get_keyword_value(ids_file_keyword) if ids_file != None and len(ids_file) > 0 and ids_file != "NONE" and sounding_ids == None: l1b_obj = acos_file.L1B(ids_file) sounding_ids = list(l1b_obj.get_sounding_ids()[:].ravel()) sounding_ids_val_sect = [] if id_list_sect == None: id_list_sect = GEN_LIST_SECTION_TMPL % run_type.upper() for list_sect in template_obj.get_section(id_list_sect + '->LIST'): list_name = list_sect.get_keyword_value('name') if list_name == SOUNDING_ID_LIST_NAME: sounding_ids_val_sect = list_sect.get_section('->VALUES') if len(sounding_ids_val_sect) == 0: raise IOError('Could not find sounding id list section named %s in %s' % (id_list_sect, template_obj.filename)) sounding_ids_val_sect[0].set_matrix_data(sounding_ids) template_obj.write(out_config_filename, doIndent=True)
def find_closest_soundings(l1b_file, tgt_latitude, tgt_longitude, max_distance, log_output=sys.stdout): l1b_obj = acos_file.L1B(l1b_file) sounding_ids = l1b_obj.get_sounding_ids() # Average over band first since this is likely to be consistent for # different types of L1B files latitudes = l1b_obj.get_sounding_info('sounding_latitude') longitudes = l1b_obj.get_sounding_info('sounding_longitude') # Average over any non sounding id sized dimensions while len(latitudes.shape) > 1: extra_dims = numpy.where(numpy.array(latitudes.shape) != sounding_ids.shape[0]) latitudes = numpy.average(latitudes, extra_dims[0][0]) longitudes = numpy.average(longitudes, extra_dims[0][0]) g = Geod(ellps='WGS84') # Find all distances in file distances = numpy.zeros(len(sounding_ids), dtype=float) for dist_idx, lat_lon_tuple in enumerate(zip(latitudes, longitudes)): curr_lat, curr_lon = lat_lon_tuple az12, az21, dist = g.inv(tgt_longitude,tgt_latitude,curr_lon,curr_lat) # Convert to km distances[dist_idx] = old_div(dist,1000.) closest = numpy.where(distances <= max_distance) if len(closest[0]) > 0: print("%s" % l1b_file, file=log_output) for close_idx in closest[0]: print('%d %f' % (sounding_ids[close_idx], distances[close_idx]), file=log_output) print("", file=log_output) else: print("No soundings found in %s closer than %f km" % (l1b_file, max_distance), file=sys.stderr)
def addHeader(options): inp_config = L2InputFile(options.config_file) l1b_file = inp_config.get_section("input->InputProductFiles")[0].Get_Keyword_Value("L1BFile") print('Using '+ l1b_file + ' to extract header information') l1b_obj = acos_file.L1B(l1b_file) l2_obj = h5py.File(options.l2_file, "r+") try: # print 'trying hard...' l2_obj.create_group("SoundingHeader") l2_obj.create_group("SoundingGeometry") except: print("Sounding header already exists in the L2 file!") sys.exit(0) exposure_index = l2_obj["RetrievalResults/exposure_index"][:]-1 sounding_id_reference = l2_obj["/RetrievalResults/sounding_id_reference"][:] time_string = l1b_obj["SoundingHeader/exposure_start_time_string"][:][exposure_index] start_time = l1b_obj["SoundingHeader/exposure_start_time_tai93"][:][exposure_index] sounding_id = l1b_obj["SoundingHeader/sounding_id"][:][exposure_index] lat = l1b_obj["SoundingGeometry/sounding_latitude"][:][exposure_index] lon = l1b_obj["SoundingGeometry/sounding_longitude"][:][exposure_index] sza = l1b_obj["SoundingGeometry/sounding_solar_zenith"][:][exposure_index] lza = l1b_obj["SoundingGeometry/sounding_zenith"][:][exposure_index] azi = l1b_obj["SoundingGeometry/sounding_solar_azimuth"][:][exposure_index] # write few entries into L2 files... l2_obj["SoundingHeader/exposure_start_time_string"] = time_string l2_obj["SoundingHeader/exposure_start_time_tai93"] = start_time l2_obj["SoundingHeader/sounding_id"] = sounding_id l2_obj["SoundingGeometry/sounding_longitude"] = lon l2_obj["SoundingGeometry/sounding_latitude"] = lat l2_obj["SoundingGeometry/sounding_solar_zenith"] = sza l2_obj["SoundingGeometry/sounding_zenith"] = lza l2_obj["SoundingGeometry/sounding_solar_azimuth"] = azi # sanity check: # print sounding_id_reference[0], sounding_id[0] l2_obj.close() l1b_obj.close() print('done...')
def mergeFiles(options): l1b_file = options.l1b_file print('Using ' + l1b_file + ' to extract header information') l1b_obj = acos_file.L1B(l1b_file) l2_obj = h5py.File(options.l2_file, "r+") try: # print 'trying hard...' l2_obj.create_group("SoundingHeader") l2_obj.create_group("SoundingGeometry") except: print("Sounding header already exists in the L2 file!") sys.exit(0) exposure_index = l2_obj["RetrievalResults/exposure_index"][:] - 1 sounding_id_reference = l2_obj[ "/RetrievalResults/sounding_id_reference"][:] time_string = l1b_obj["SoundingHeader/exposure_start_time_string"][:][ exposure_index] try: start_time = l1b_obj["SoundingHeader/exposure_start_time_tai93"][:][ exposure_index] except: print( "exposure_start_time not found (probably simulator file?), skipping this..." ) # write few entries into L2 files... l2_obj["SoundingHeader/exposure_start_time_string"] = time_string try: l2_obj["SoundingHeader/exposure_start_time_tai93"] = start_time except: print("time missing") l2_obj["SoundingHeader/sounding_id"] = l1b_obj[ "SoundingHeader/sounding_id"][:][exposure_index] l2_obj["SoundingGeometry/sounding_longitude"] = l1b_obj[ "SoundingGeometry/sounding_longitude"][:][exposure_index] l2_obj["SoundingGeometry/sounding_latitude"] = l1b_obj[ "SoundingGeometry/sounding_latitude"][:][exposure_index] l2_obj["SoundingGeometry/sounding_solar_zenith"] = l1b_obj[ "SoundingGeometry/sounding_solar_zenith"][:][exposure_index] l2_obj["SoundingGeometry/sounding_zenith"] = l1b_obj[ "SoundingGeometry/sounding_zenith"][:][exposure_index] l2_obj["SoundingGeometry/sounding_solar_azimuth"] = l1b_obj[ "SoundingGeometry/sounding_solar_azimuth"][:][exposure_index] l2_obj["SoundingGeometry/sounding_azimuth"] = l1b_obj[ "SoundingGeometry/sounding_azimuth"][:][exposure_index] l2_obj["SoundingGeometry/sounding_altitude"] = l1b_obj[ "FootprintGeometry/footprint_altitude"][:][exposure_index] # sanity check: # print sounding_id_reference[0], sounding_id[0] ################################################################################### # Proceed with the log file: l2_obj.create_group("truth") l2_obj.create_group("truth/log") l2_obj.create_group("truth/scene") l2_obj.create_group("truth/simulation") logData = loadtxt(options.log_file, comments='#') l2_obj["truth/log/frame"] = int32(logData[exposure_index, 0]) l2_obj["truth/log/sounding_id"] = int64(logData[exposure_index, 1]) l2_obj["truth/log/surface_model"] = logData[exposure_index, 2] l2_obj["truth/log/albedo_1A"] = logData[exposure_index, 3] l2_obj["truth/log/albedo_1B"] = logData[exposure_index, 4] l2_obj["truth/log/albedo_2A"] = logData[exposure_index, 5] l2_obj["truth/log/albedo_2B"] = logData[exposure_index, 6] l2_obj["truth/log/albedo_3A"] = logData[exposure_index, 7] l2_obj["truth/log/albedo_3B"] = logData[exposure_index, 8] l2_obj["truth/log/tau_water_1"] = logData[exposure_index, 9] l2_obj["truth/log/tau_ice_1"] = logData[exposure_index, 10] l2_obj["truth/log/tau_aerosol_1"] = logData[exposure_index, 11] l2_obj["truth/log/tau_water_2"] = logData[exposure_index, 12] l2_obj["truth/log/tau_ice_2"] = logData[exposure_index, 13] l2_obj["truth/log/tau_aerosol_2"] = logData[exposure_index, 14] l2_obj["truth/log/tau_water_3"] = logData[exposure_index, 15] l2_obj["truth/log/tau_ice_3"] = logData[exposure_index, 16] l2_obj["truth/log/tau_aerosol_3"] = logData[exposure_index, 17] l2_obj["truth/log/logFile"] = options.log_file ################################################################################### # Proceed with the scene log file: scene = loadtxt(options.scene_file, comments='#') l2_obj["truth/scene/sceneFile"] = options.scene_file l2_obj["truth/scene/frame"] = int32(scene[exposure_index, 0]) l2_obj["truth/scene/sounding_id"] = int64(scene[exposure_index, 1]) l2_obj["truth/scene/sounding"] = int32(scene[exposure_index, 2]) l2_obj["truth/scene/profile_index"] = scene[exposure_index, 3] l2_obj["truth/scene/sat_latitude"] = scene[exposure_index, 4] l2_obj["truth/scene/sat_longitude"] = scene[exposure_index, 5] l2_obj["truth/scene/fov_latitude"] = scene[exposure_index, 6] l2_obj["truth/scene/fov_longitude"] = scene[exposure_index, 7] l2_obj["truth/scene/sun_latitude"] = scene[exposure_index, 8] l2_obj["truth/scene/sun_longitude"] = scene[exposure_index, 9] l2_obj["truth/scene/XCO2"] = scene[exposure_index, 10] l2_obj["truth/scene/psurf"] = scene[exposure_index, 11] l2_obj["truth/scene/mean_temperature"] = scene[exposure_index, 12] l2_obj["truth/scene/cloud_water"] = scene[exposure_index, 13] l2_obj["truth/scene/cloud_ice"] = scene[exposure_index, 14] l2_obj["truth/scene/scattering_height_water"] = scene[exposure_index, 15] l2_obj["truth/scene/scattering_height_ice"] = scene[exposure_index, 16] l2_obj["truth/scene/aerosol_1"] = scene[exposure_index, 17] l2_obj["truth/scene/aerosol_2"] = scene[exposure_index, 18] l2_obj["truth/scene/aerosol_3"] = scene[exposure_index, 19] l2_obj["truth/scene/aerosol_4"] = scene[exposure_index, 20] l2_obj["truth/scene/aerosol_5"] = scene[exposure_index, 21] l2_obj["truth/scene/aerosol_6"] = scene[exposure_index, 22] l2_obj["truth/scene/aerosol_7"] = scene[exposure_index, 23] l2_obj["truth/scene/sun_zenith"] = scene[exposure_index, 24] l2_obj["truth/scene/sat_zenith"] = scene[exposure_index, 25] l2_obj["truth/scene/IGBP_index"] = scene[exposure_index, 26] l2_obj["truth/scene/surface_windSpeed"] = scene[exposure_index, 27] l2_obj["truth/scene/moist_air"] = scene[exposure_index, 28] l2_obj["truth/scene/h2o"] = scene[exposure_index, 29] l2_obj["truth/scene/h2o_"] = scene[exposure_index, 30] # l2_obj["truth/scene/h2o_"].attrs['Units']='kg/m^2' ################################################################################### # Proceed with the detailed HDF scene file sim = h5py.File(options.logHDF_file, "r") l2_obj.create_group("truth/simulation/Aerosol") l2_obj.create_group("truth/simulation/Gas") l2_obj.create_group("truth/simulation/Surface") l2_obj.create_group("truth/simulation/Thermodynamic") l2_obj["truth/simulation/Aerosol/num_species"] = sim[ "Simulation/Aerosol/num_species"][:][exposure_index] l2_obj["truth/simulation/Aerosol/species_density"] = sim[ "Simulation/Aerosol/species_density"][:][exposure_index, :, :] l2_obj["truth/simulation/Aerosol/species_id"] = sim[ "Simulation/Aerosol/species_id"][:][exposure_index, :] l2_obj["truth/simulation/Gas/num_species"] = sim[ "Simulation/Gas/num_species"][:][exposure_index] l2_obj["truth/simulation/Gas/species_density"] = sim[ "Simulation/Gas/species_density"][:][exposure_index, :, :] l2_obj["truth/simulation/Gas/species_id"] = sim[ "Simulation/Gas/species_id"][:][exposure_index, :] l2_obj["truth/simulation/Surface/land_fraction"] = sim[ "Simulation/Surface/land_fraction"][:][exposure_index] l2_obj["truth/simulation/Thermodynamic/altitude_level"] = sim[ "Simulation/Thermodynamic/altitude_level"][:][exposure_index, :] l2_obj["truth/simulation/Thermodynamic/num_layers"] = sim[ "Simulation/Thermodynamic/num_layers"][:][exposure_index] l2_obj["truth/simulation/Thermodynamic/pressure_level"] = sim[ "Simulation/Thermodynamic/pressure_level"][:][exposure_index, :] l2_obj["truth/simulation/Thermodynamic/temperature_level"] = sim[ "Simulation/Thermodynamic/temperature_level"][:][exposure_index, :] ################################################################################### # Try to implement the averaging kernel correction (all done on the retrieval grid, not the simulator grid, to be though over) l2_obj.create_group("truth/co2_profile") press_level = l2_obj["RetrievalResults/vector_pressure_levels"][:] n_level = l2_obj["RetrievalResults/num_active_levels"][:] l2_obj["truth/co2_profile/p"] = l2_obj[ "RetrievalResults/vector_pressure_levels"] l2_obj["truth/co2_profile/n_levels"] = l2_obj[ "RetrievalResults/num_active_levels"] l2_obj["truth/co2_profile/co2_retrieved"] = l2_obj[ "RetrievalResults/co2_profile"] l2_obj["truth/co2_profile/co2_apriori"] = l2_obj[ "RetrievalResults/co2_profile_apriori"] # generate true profiles interpolated on the retrieval grid (has to be extrapolated at times, stupid!) co2_prior = l2_obj["RetrievalResults/co2_profile_apriori"] p_coarse = l2_obj["RetrievalResults/vector_pressure_levels"] n_coarse = l2_obj["RetrievalResults/num_active_levels"] n_fine = sim["Simulation/Thermodynamic/num_layers"][:][exposure_index] p = sim["Simulation/Thermodynamic/pressure_level"][:][exposure_index, :] gases = sim["Simulation/Gas/species_density"][:][exposure_index, :, :] AK_all = l2_obj["/RetrievalResults/averaging_kernel_matrix"] h = l2_obj["RetrievalResults/xco2_pressure_weighting_function"] s = (len(exposure_index), 20) co2_interp = zeros(s, dtype='f') co2_ak_corrected = zeros(s, dtype='f') xCO2_true = zeros(len(exposure_index), dtype='f') xCO2_true_AK = zeros(len(exposure_index), dtype='f') # loop over all entries for i in range(len(exposure_index)): # print p[i,:n_fine[i]],gases[i,3,:n_fine[i]]/gases[i,1,:n_fine[i]] co2_interp[i, :n_coarse[i]] = interp( p_coarse[i, :n_coarse[i]], p[i, :n_fine[i]], old_div(gases[i, 3, :n_fine[i]], gases[i, 1, :n_fine[i]])) AK_co2 = AK_all[i, :n_coarse[i], :n_coarse[i]] co2_ak_corrected[i, :n_coarse[i]] = co2_prior[i, :n_coarse[i]] + inner( AK_co2, co2_interp[i, :n_coarse[i]] - co2_prior[i, :n_coarse[i]]) xCO2_true[i] = dot(h[i, :n_coarse[i]].transpose(), co2_interp[i, :n_coarse[i]]) * 1e6 xCO2_true_AK[i] = dot(h[i, :n_coarse[i]].transpose(), co2_ak_corrected[i, :n_coarse[i]]) * 1e6 l2_obj["truth/co2_profile/co2_truth"] = co2_interp l2_obj["truth/co2_profile/xco2_truth_calc"] = xCO2_true l2_obj["truth/co2_profile/xco2_truth_calc_AK_applied"] = xCO2_true_AK l2_obj["truth/co2_profile/co2_truth_AK_applied"] = co2_ak_corrected ################################################################################### # Use the aerosol optical depth profiles from the opt files. opt = h5py.File(options.opt_file, "r") aod = opt["Simulation/Aerosol/optical_thickness"][:][ exposure_index, :, :, :] species = opt["Simulation/Aerosol/species_id"][:][exposure_index, :] num_aero = opt["Simulation/Aerosol/num_species"][:][exposure_index] p_hr = l2_obj["truth/simulation/Thermodynamic/pressure_level"][:] n_hr = sim["Simulation/Thermodynamic/num_layers"][:][exposure_index] a = aod.shape od_total = zeros((a[0], a[2], a[3]), dtype='f') od_ice = zeros((a[0], a[2], a[3]), dtype='f') od_water = zeros((a[0], a[2], a[3]), dtype='f') od_aerosol = zeros((a[0], a[2], a[3]), dtype='f') od_total_lr = zeros((a[0], a[2], 19), dtype='f') od_ice_lr = zeros((a[0], a[2], 19), dtype='f') od_water_lr = zeros((a[0], a[2], 19), dtype='f') od_aerosol_lr = zeros((a[0], a[2], 19), dtype='f') p_hr2 = old_div((p_hr[:, 1:] + p_hr[:, 0:-1]), 2) for i in range(a[0]): od_total[i, :, :] = aod[i, :, :, :].sum(axis=0) ind_water = zeros(0, dtype='i') ind_ice = zeros(0, dtype='i') ind_aero = zeros(0, dtype='i') # Look for different aerosol types and find indices... for typ_id in range(num_aero[i]): # print species[i,typ_id][0:3] if species[i, typ_id][0:3] == 'wat': ind_water = append(ind_water, typ_id) elif species[i, typ_id][0:3] == 'ice': ind_ice = append(ind_ice, typ_id) else: ind_aero = append(ind_aero, typ_id) # Look for different profiles to be added into retrieval layers... for press_id in range(n_level[i] - 1): ind = ((p_hr2[i, :] >= press_level[i, press_id]) & (p_hr2[i, :] < press_level[i, press_id + 1])).nonzero() for spec in range(a[2]): od_total_lr[i, spec, press_id] = aod[i, :, spec, ind].sum() if len(ind_water) > 0: for ind_a in ind_water: od_water_lr[i, spec, press_id] += aod[i, ind_a, spec, ind].sum() if len(ind_aero) > 0: for ind_a in ind_aero: od_aerosol_lr[i, spec, press_id] += aod[i, ind_a, spec, ind].sum() if len(ind_ice) > 0: for ind_a in ind_ice: od_ice_lr[i, spec, press_id] = aod[i, ind_a, spec, ind].sum() od_water[i, :, :] = aod[i, ind_water, :, :].sum(axis=0) od_ice[i, :, :] = aod[i, ind_ice, :, :].sum(axis=0) od_aerosol[i, :, :] = aod[i, ind_aero, :, :].sum(axis=0) # print "1: ", od_aerosol[i,0,:].sum(), od_aerosol[i,1,:].sum(), l2_obj["truth/log/tau_aerosol_1"][i] # print "3: ", od_aerosol[i,4,:].sum(), od_aerosol_lr[i,4,:].sum(), l2_obj["truth/log/tau_aerosol_3"][i] l2_obj["truth/simulation/Aerosol/optical_thickness"] = aod l2_obj["truth/simulation/Aerosol/optical_thickness_all"] = od_total l2_obj["truth/simulation/Aerosol/optical_thickness_water"] = od_water l2_obj["truth/simulation/Aerosol/optical_thickness_ice"] = od_ice l2_obj["truth/simulation/Aerosol/optical_thickness_aero"] = od_aerosol l2_obj[ "truth/simulation/Aerosol/optical_thickness_all_lowRes"] = od_total_lr l2_obj[ "truth/simulation/Aerosol/optical_thickness_water_lowRes"] = od_water_lr l2_obj["truth/simulation/Aerosol/optical_thickness_ice_lowRes"] = od_ice_lr l2_obj[ "truth/simulation/Aerosol/optical_thickness_aero_lowRes"] = od_aerosol_lr ################################################################################### # Do the cloud screen stuff. if options.cloud_file == None: print("will skip the A-band cloud screen results") else: l2_obj.create_group("ABandCloudScreen") cld = h5py.File(options.cloud_file, "r") aband = cld["ABandCloudScreen"] for data in list(aband.items()): if len(data[1].shape) == 1: l2_obj['ABandCloudScreen/' + data[0]] = data[1][:][exposure_index] elif len(data[1].shape) == 2: l2_obj['ABandCloudScreen/' + data[0]] = data[1][:][exposure_index, :] cld.close() opt.close() sim.close() l2_obj.close() l1b_obj.close() print('done...')