def _get_num_iter_from_output_path(self, workdir):
'''
Determine the number of iterations based on either the namelist or the output files
# TODO : need a unit test for this (e.g. different gsiparm.anl files: miter=2, miter=0, no miter)
'''
# try determining from namelist option miter
with open(os.path.join(workdir, 'gsiparm.anl')) as elFile:
for line in elFile.readlines():
if line.find("miter") > 0:
toks = line.split("=")
miter = int(toks[1][0])
if miter == 2: # assuming no characters before the value
log.info("Guessing this is an analysis run based on namelist value of 'miter'")
return 2
else:
log.info("Guessing this is an O-F only run based on namelist value of 'miter'")
return 0
#try to find 'anl' diag files
pattern = DIAG_FILE_PREFIX + "*" + "_" + ANL_DIAGFILE_SUFFIX + "." + epoch_to_yyyymmddHHMM(self.date) + DIAG_FILE_SUFFIX
matches = glob.glob(os.path.join(workdir, pattern) )
if len(matches) > 0:
log.info("Guessing this is an analysis run based on existence of diag files for analysis")
return 2
# now try with the plaintext diag files
pattern = TEXT_DIAG_PREFIX + "*" + "_" + ANL_DIAGFILE_SUFFIX + "." + epoch_to_yyyymmddHHMM(self.date) + TEXT_DIAG_SUFFIX
matches = glob.glob(os.path.join(workdir, pattern) )
if len(matches) > 0:
log.info("Guessing this is an analysis run based on existence of text diag files for analysis")
return 2
# assume its ges only
return 0
def _get_obTypes_from_products_dir(self):
'''
RETURN A list of obTypes by looking in self.products_dir for files matching the diag
file naming convention.
First, look for the binary diag files. If none are found, look for the plaintext
diag files. If none are found, raise an exception
If this is an analysis experiment, use the "_anl" files to get the obTypes.
Otherwise use the "_ges" files
TODO: Create unit test for this (1)with binary diag files (2) plaintext diag files
'''
if self.is_analysis:
diag_step = ANL_DIAGFILE_SUFFIX
log.info("Since this is an analysis run, will only process ob types for which a diag file was created for the analysis")
else:
diag_step = GES_DIAGFILE_SUFFIX
file_prefix = DIAG_FILE_PREFIX
file_suffix = "_" + diag_step + "." + epoch_to_yyyymmddHHMM(self.date) + DIAG_FILE_SUFFIX
pattern = file_prefix + "*" + file_suffix # diag_cris_npp_anl.200508010600
log.debug("Looking for files matching pattern %s" %os.path.join(self.products_dir, pattern))
diag_files = glob.glob( os.path.join(self.products_dir, pattern) )
if len(diag_files) > 0:
log.debug("Using GSI-generated binary diagnostic files")
return [ x[x.index(file_prefix)+len(file_prefix):x.index(file_suffix)] for x in diag_files ]
else: # attempt to find plaintext diag files
file_prefix = TEXT_DIAG_PREFIX
file_suffix = "_" + diag_step + "." + epoch_to_yyyymmddHHMM(self.date) + TEXT_DIAG_SUFFIX
pattern = file_prefix + "*" + diag_step + file_suffix
diag_files = glob.glob( os.path.join(self.products_dir, pattern ) )
if len(diag_files) > 0:
self.use_plaintext_diag = True
log.debug("Using plaintext diagnostic files")
return [ x[x.index(len(file_prefix)):x.index(file_suffix)] for x in diag_files ]
else:
raise Exception("Could not find any diagnostic files")
def get_diag_file_name(obType, diag_step, date, is_text_diag):
'''
Create the file name based on known GSI naming convention
ARGS:
obType - The ob name or <obType>_<platform> "tag" used in GSI naming convention
diag_step - the keyword used by GSI to distinguish ges and analysis (i.e. 'ges' or 'anl')
date - The cycle date in seconds since epoch
is_text_diag - True if using plaintext diag files
'''
if is_text_diag:
return TEXT_DIAG_PREFIX + obType + "_" + diag_step + "." + epoch_to_yyyymmddHHMM(date) + TEXT_DIAG_SUFFIX
else:
return DIAG_FILE_PREFIX + obType + "_" + diag_step + "." + epoch_to_yyyymmddHHMM(date) + DIAG_FILE_SUFFIX
def make_plain_text_diag_file(self, outfile='', outdir='', gz=True, include_skipped=False):
'''
Make a plain-text diagnostic file given a GSI-generated "binary" diagnostic file
PARAMETERS
- outfile - Specify an output file name. If none is chosen, use "PLAIN_TEXT_DIAG_PREFIX + "_" + diag_step + "." + epoch_to_yyyymmddHHMM(cycle) + TEXT_DIAG_SUFFIX"
- outdir - Specify and output directory. If none is given, use CWD
- gz (Default True) - zip the output file using gzip
- include_skipped (Default False) - include skipped (i.e. iuse != 1) observations in the output
'''
#import pdb ; pdb.set_trace()
if len(outdir) == 0:
outdir = os.getcwd()
if len(outfile) == 0:
outfile = TEXT_DIAG_PREFIX + self.ob_type + "_" + self.step + "." + epoch_to_yyyymmddHHMM(self.date) + TEXT_DIAG_SUFFIX
if gz:
outfile = gzip.open( os.path.join(outdir, outfile), 'wb')
else:
outfile = open( os.path.join(dataset.get_gsi_output_path(cycle), outfile), 'w')
if include_skipped:
#all_obs = self.conv_obs + self.rad_obs + self.oz_obs
all_obs = self.obs
else:
#all_obs = self.non_skipped_conv_obs + self.non_skipped_rad_obs + self.non_skipped_oz_obs
all_obs = [ ob for ob in self.obs if not ob.skipped ]
for ob in all_obs:
outfile.write( ob.to_csv() + "\n")
def _extract_gsi_diag(self):
'''
Read GSI diag file using a reader, convert entries into GsiObservation objects, and return
a list containing these objects
'''
#in_file_name = 'diag_' + ob_string + '_' + diag_step + '.' + epoch_to_yyyymmddHHMM(cycle)
intermediate_file_name = 'results_' + self.ob_type + '_' + self.step + '.' + epoch_to_yyyymmddHHMM(
self.date)
# determine ob type
log.debug("Processing ob type: %s for step %s" %
(self.ob_type, self.step))
# create intermediate text file
try:
owd = os.getcwd()
os.chdir(self.path)
except:
log.error("Unable to change to directory: %s" % self.path)
sys.exit(13)
self._create_diag_reader_namelist(self.ob_type, self.diag_file_name,
intermediate_file_name)
if self.ob_type == 'conv':
ret = subprocess.call([self.gsi_diag_reader_conv],
shell=True,
stdout=open("/dev/null", 'w'))
elif self.ob_type == 'oz':
raise Exception("Not implemented for oz obs")
else: # assume radiance
ret = subprocess.call([self.gsi_diag_reader_rad],
shell=True,
stdout=open("/dev/null", 'w'))
# TODO ensure it returns with exit code 9999 AND have option of sending output to a file instead of /dev/null
# process output
if self.ob_type == 'conv':
obs = self.process_conv_diag_reader_output(intermediate_file_name)
else: # assume rad
obs = self.process_rad_diag_reader_output(intermediate_file_name)
try:
os.chdir(owd)
except:
log.error("Unable to switch back to original directory: %s" % owd)
sys.exit(13)
finally:
os.unlink(os.path.join(self.path, intermediate_file_name))
return obs