def read_run_id_file(run_id_file):
# Use L2_Input to parse XML / heritage format file
# as well as a regular ascii file
parsed_file = L2InputFile(run_id_file)
section_names = parsed_file.get_all_section_names()
# If there are section names then this is an XML or heritage format
# used to drive the populator
if len(section_names) > 0:
# Try to find section where run ids live
list_sections = [sec_name for sec_name in section_names if re.search(CONFIG_RUN_ID_SEC_RE, sec_name)]
id_lines = []
for sec_name in list_sections:
found_sec_objs = parsed_file.get_section(sec_name)
if len(found_sec_objs) > 0:
id_lines += found_sec_objs[0].get_matrix_data()
else:
# Otherwise this is just a plain ascii file we
# have parsed into a list of lists
id_lines = parsed_file.get_matrix_data()
# Pull run ids out of list of lists parsed above
run_id_list = []
for id_line in id_lines:
# Parse out first part of space seperated line string or first item from list
if isinstance(id_line, str):
run_id = id_line.split()[0].strip()
elif hasattr(id_line, "__iter__"):
run_id = id_line[0]
run_id_list.append(run_id)
return run_id_list
def convert_fts_data(fts_run_dir,
output_filename,
l1b_filename=REFERENCE_L1B_FILE,
config_filename=DEFAULT_FTS_CONFIG_FILE,
window_filename=DEFAULT_FTS_CONFIG_FILE,
sounding_pos=DEFAULT_SOUNDING):
# Open files needed for processing
output_file = h5py.File(output_filename, 'w')
l1b_file = h5py.File(l1b_filename, 'r')
# Get per sounding information from files in the populated FTS run directory
obs_ids = [
int(sid)
for sid in np.loadtxt(os.path.join(fts_run_dir, 'sounding_id.list'))
]
spec_a_files = [
l.strip() for l in open(os.path.join(fts_run_dir, 'spectrum_a.list'),
'r').readlines()
]
spec_b_files = [
l.strip() for l in open(os.path.join(fts_run_dir, 'spectrum_b.list'),
'r').readlines()
]
# Initialize output file datasets
create_datasets(obs_ids, output_file, l1b_file)
# Load the run directory configuration file to determine location of run log
config_fn = glob(os.path.join(fts_run_dir, '*.config'))[0]
config_file = L2InputFile(config_fn)
config_sec = config_file.rootNode.get_section(
'input->InputProductFiles')[0]
runlog_file = config_sec.get_keyword_value('RunlogFile')
logger.debug('Run Log = %s' % runlog_file)
spec_win = spectral_window(window_filename)
# Load the FTS L1B object for each sounding, writing that information to the output file
for idx, (obs_id, spec_a,
spec_b) in enumerate(zip(obs_ids, spec_a_files, spec_b_files)):
spectra_names = [spec_b, spec_a, spec_a]
l1b_fts = Level1bFts(runlog_file, spectra_names,
spec_win.spectral_bound)
sounding_id = create_sounding_id(obs_id, l1b_fts, sounding_pos)
logger.debug('Obs ID = %s, Sounding ID = %s' % (obs_id, sounding_id))
logger.debug('Spectrum A = %s' % spec_a)
logger.debug('Spectrum B = %s' % spec_b)
store_l1b_data(sounding_id,
idx,
l1b_fts,
output_file,
l1b_file,
sounding_pos=sounding_pos)
def create_from_existing_run(config_file, sounding_id = None,
sounding_id_index = 0, lua_config = None):
'''This creates a L2Run object that sets everything up from the
an existing L2 run. We take the configuration file that was
passed to the populator script, and either the sounding ID, or
the index into the list of soundings IDs (e.g., 0 for the first index,
etc.)
Since it is often useful, we stash the output file that we use into
the results as "output_file"
The state vector is in the state set by the Lua configuration.
You may man to look at create_final_state and create_initial_state
which are closely related functions that set the state vector into
a known state.
Default lua_config is the normal gosat/config/config.lua, but you can
specify a different one if desired
'''
c = L2InputFile(config_file)
t = c.get_section("input->InputProductFiles")[0]
l1b = t.get_keyword_value("L1BFile")
met = t.get_keyword_value("ResampledMetFile")
co2_pr = t.get_keyword_value("CO2PriorFile")
imap = t.get_keyword_value("IMAPFile")
if(lua_config is None):
lua_config = os.path.join(os.environ.get('L2_INPUT_PATH', ''), "gosat/config/config.lua")
if(sounding_id is None):
t = c.get_section("input->GOSATFullPhysics->LIST->VALUES")
if(len(t) == 0):
t = c.get_section("input->OCOFullPhysics->LIST->VALUES")
sid = t[0].get_matrix_data()
sounding_id = sid[sounding_id_index]
if(t.get_keyword_value("InputFileMapping")):
inpmap = t.get_keyword_value("InputFileMapping")
with open(inpmap) as f:
for ln in f:
sid,val = ln.split(' ')
if(sid == sounding_id):
for v2 in val.split(';'):
k,v = v2.split('=')
if(k == "spectrum_file"):
l1b = eval(v)
if(k == "met_file"):
met = eval(v)
if(k == "co2_pr_file"):
co2_pr = eval(v)
if(k == "imap_file"):
imap = eval(v)
r = L2Run(lua_config, sounding_id, met, l1b, imap_file = imap,
co2_pr_file = co2_pr)
r.output_file_name = os.path.dirname(config_file) + "/output/l2_" + \
sounding_id + ".h5"
return r
def create_populator_from_config_file(config_file, **user_settings):
'''Read the L2 input configuration file supplied, and based on the
type return the populator object that matches the type. If we can't
find it, return None.'''
l2_config = L2InputFile(config_file)
t = l2_config.get_section("input->PGENameGroup")
if len(t) > 0:
pge = t[0].get_keyword_value("PGEName")
if (pge is not None and len(pge) > 0):
for k, v in list(PopulatorBase.populator_list.items()):
t = v(**user_settings)
if (re.search(pge, t.config_file_obj_re)):
return t
# Some older config files don't have the PGE in it, so try working
# off of the file name
for k, v in list(PopulatorBase.populator_list.items()):
if (re.search(config_file, r'^%s_' % k)):
return v(**user_settings)
return None
def addHeader(options):
inp_config = L2InputFile(options.config_file)
l1b_file = inp_config.get_section("input->InputProductFiles")[0].Get_Keyword_Value("L1BFile")
print('Using '+ l1b_file + ' to extract header information')
l1b_obj = acos_file.L1B(l1b_file)
l2_obj = h5py.File(options.l2_file, "r+")
try:
# print 'trying hard...'
l2_obj.create_group("SoundingHeader")
l2_obj.create_group("SoundingGeometry")
except:
print("Sounding header already exists in the L2 file!")
sys.exit(0)
exposure_index = l2_obj["RetrievalResults/exposure_index"][:]-1
sounding_id_reference = l2_obj["/RetrievalResults/sounding_id_reference"][:]
time_string = l1b_obj["SoundingHeader/exposure_start_time_string"][:][exposure_index]
start_time = l1b_obj["SoundingHeader/exposure_start_time_tai93"][:][exposure_index]
sounding_id = l1b_obj["SoundingHeader/sounding_id"][:][exposure_index]
lat = l1b_obj["SoundingGeometry/sounding_latitude"][:][exposure_index]
lon = l1b_obj["SoundingGeometry/sounding_longitude"][:][exposure_index]
sza = l1b_obj["SoundingGeometry/sounding_solar_zenith"][:][exposure_index]
lza = l1b_obj["SoundingGeometry/sounding_zenith"][:][exposure_index]
azi = l1b_obj["SoundingGeometry/sounding_solar_azimuth"][:][exposure_index]
# write few entries into L2 files...
l2_obj["SoundingHeader/exposure_start_time_string"] = time_string
l2_obj["SoundingHeader/exposure_start_time_tai93"] = start_time
l2_obj["SoundingHeader/sounding_id"] = sounding_id
l2_obj["SoundingGeometry/sounding_longitude"] = lon
l2_obj["SoundingGeometry/sounding_latitude"] = lat
l2_obj["SoundingGeometry/sounding_solar_zenith"] = sza
l2_obj["SoundingGeometry/sounding_zenith"] = lza
l2_obj["SoundingGeometry/sounding_solar_azimuth"] = azi
# sanity check:
# print sounding_id_reference[0], sounding_id[0]
l2_obj.close()
l1b_obj.close()
print('done...')
def get_config_keyword_value(self, config_filename, keyword_path):
'''Read a L2 input file as keyword/value pairs, and return the value
for the given keyword'''
config_obj = L2InputFile(config_filename)
search_sect_name = '->'.join(keyword_path.split('/')[0:-1])
search_key_name = keyword_path.split('/')[-1]
search_sect_obj = config_obj.get_section(search_sect_name)
if search_sect_obj == None or len(search_sect_obj) == 0:
raise KeyError('Could not find section: %s in file: %s' %
(search_sect_name, config_filename))
found_values = [
sect.get_keyword_value(search_key_name) for sect in search_sect_obj
]
if len(found_values) == 1:
return found_values[0]
else:
return found_values
def read_id_list_file(self, id_list_file, section=None):
'''Read the sounding ids from the given list of files. The file
list can contain wildcards and/or ~'''
# Use old L2_Input syntax
if section != None:
section = section.replace("/", "->")
if section != None:
self.logger.debug('Reading id list from section %s file: %s' %
(section, id_list_file))
id_list_str = self.__get_list_file_values(id_list_file,
str(id_list_file),
section)
else:
self.logger.debug('Reading id list from file: %s' % id_list_file)
id_obj = L2InputFile(id_list_file)
id_list_str = id_obj.get_matrix_data()
if id_list_str == None:
return []
id_list_long = []
for curr_id_str in id_list_str:
# Try to match sounding id pattern
id_match = re.search('\d{3,17}\w?', curr_id_str)
if not id_match:
raise IOError(
'Could not find sounding id in string: "%s" in file %s' %
(curr_id_str, id_list_file))
beg_pos = int(id_match.start())
end_pos = int(id_match.end())
found_id = curr_id_str[beg_pos:end_pos]
id_list_long.append(found_id)
return id_list_long
def __get_list_file_values(self,
listLocation,
listName,
sectionName=None,
directoryLevels=None):
# Expand any globs
if isinstance(listLocation, six.string_types):
listLocation = self.__expand_filename(listLocation)
# Make sure we have something to iterate over
elif not hasattr(listLocation, '__iter__'):
listLocation = [listLocation]
if directoryLevels == None:
directoryLevels = 1
else:
if not type(
directoryLevels) is int and not directoryLevels.isdigit():
raise ValueError(
'directoryLevels argument must be an integer or convertable to one'
)
else:
directoryLevels = int(directoryLevels)
fileValues = []
for listFile in listLocation:
if isinstance(listFile,
six.string_types) and os.path.isdir(listFile):
self.logger.debug(
'Loading LIST %s contents from directory: %s' %
(listName, listFile))
# Use directoryLevels - 1 parts of the end of the path
old_path = listFile
new_path = ''
for level_idx in range(directoryLevels - 1):
old_path, dir_name = os.path.split(old_path)
new_path = os.path.join(new_path, dir_name)
# Use path as a list based on filenames present there
for dir_filename in os.listdir(listFile):
fileValues.append(os.path.join(new_path, dir_filename))
elif sectionName == None:
self.logger.debug('Loading LIST %s contents from file: %s' %
(listName, listFile))
if isinstance(listFile, six.string_types):
listFileObj = open(listFile, 'r')
elif hasattr(listFile, 'read'):
listFileObj = listFile
else:
raise Exception('Unknown read object: %s' % listFile)
for listLine in listFileObj.readlines():
if len(listLine.strip()) > 0 and listLine.strip(
)[0] != '#':
fileValues.append(listLine.strip())
if isinstance(listFile, six.string_types):
listFileObj.close()
else:
self.logger.debug(
'Loading LIST %s section as %s contents from file: %s' %
(sectionName, listName, listFile))
fileObj = L2InputFile(listFile)
sectNameParts = sectionName.split('->')
foundSects = fileObj.get_section(
'->'.join(sectNameParts[0:-1]) + '->LIST')
fileListSect = None
for currFileSect in foundSects:
currListName = currFileSect.get_keyword_value('name')
if currListName != None and currListName == sectNameParts[
-1]:
fileListSect = currFileSect.get_section('LIST->VALUES')
break
if fileListSect == None or len(fileListSect) == 0:
raise IOError('Could not find section %s in file: %s' %
(sectionName, listFile))
fileValues = fileListSect[0].get_matrix_data()
return fileValues
def _l2_input_file(self, file):
'''This caches reading a config file, so we don't parse the same file
multiple_times.'''
if (file not in self._l2_input_file_cache):
self._l2_input_file_cache[file] = L2InputFile(file)
return self._l2_input_file_cache[file]