def _get_num_iter_from_output_path(self, workdir): ''' Determine the number of iterations based on either the namelist or the output files # TODO : need a unit test for this (e.g. different gsiparm.anl files: miter=2, miter=0, no miter) ''' # try determining from namelist option miter with open(os.path.join(workdir, 'gsiparm.anl')) as elFile: for line in elFile.readlines(): if line.find("miter") > 0: toks = line.split("=") miter = int(toks[1][0]) if miter == 2: # assuming no characters before the value log.info("Guessing this is an analysis run based on namelist value of 'miter'") return 2 else: log.info("Guessing this is an O-F only run based on namelist value of 'miter'") return 0 #try to find 'anl' diag files pattern = DIAG_FILE_PREFIX + "*" + "_" + ANL_DIAGFILE_SUFFIX + "." + epoch_to_yyyymmddHHMM(self.date) + DIAG_FILE_SUFFIX matches = glob.glob(os.path.join(workdir, pattern) ) if len(matches) > 0: log.info("Guessing this is an analysis run based on existence of diag files for analysis") return 2 # now try with the plaintext diag files pattern = TEXT_DIAG_PREFIX + "*" + "_" + ANL_DIAGFILE_SUFFIX + "." + epoch_to_yyyymmddHHMM(self.date) + TEXT_DIAG_SUFFIX matches = glob.glob(os.path.join(workdir, pattern) ) if len(matches) > 0: log.info("Guessing this is an analysis run based on existence of text diag files for analysis") return 2 # assume its ges only return 0
def _get_obTypes_from_products_dir(self): ''' RETURN A list of obTypes by looking in self.products_dir for files matching the diag file naming convention. First, look for the binary diag files. If none are found, look for the plaintext diag files. If none are found, raise an exception If this is an analysis experiment, use the "_anl" files to get the obTypes. Otherwise use the "_ges" files TODO: Create unit test for this (1)with binary diag files (2) plaintext diag files ''' if self.is_analysis: diag_step = ANL_DIAGFILE_SUFFIX log.info("Since this is an analysis run, will only process ob types for which a diag file was created for the analysis") else: diag_step = GES_DIAGFILE_SUFFIX file_prefix = DIAG_FILE_PREFIX file_suffix = "_" + diag_step + "." + epoch_to_yyyymmddHHMM(self.date) + DIAG_FILE_SUFFIX pattern = file_prefix + "*" + file_suffix # diag_cris_npp_anl.200508010600 log.debug("Looking for files matching pattern %s" %os.path.join(self.products_dir, pattern)) diag_files = glob.glob( os.path.join(self.products_dir, pattern) ) if len(diag_files) > 0: log.debug("Using GSI-generated binary diagnostic files") return [ x[x.index(file_prefix)+len(file_prefix):x.index(file_suffix)] for x in diag_files ] else: # attempt to find plaintext diag files file_prefix = TEXT_DIAG_PREFIX file_suffix = "_" + diag_step + "." + epoch_to_yyyymmddHHMM(self.date) + TEXT_DIAG_SUFFIX pattern = file_prefix + "*" + diag_step + file_suffix diag_files = glob.glob( os.path.join(self.products_dir, pattern ) ) if len(diag_files) > 0: self.use_plaintext_diag = True log.debug("Using plaintext diagnostic files") return [ x[x.index(len(file_prefix)):x.index(file_suffix)] for x in diag_files ] else: raise Exception("Could not find any diagnostic files")
def get_diag_file_name(obType, diag_step, date, is_text_diag): ''' Create the file name based on known GSI naming convention ARGS: obType - The ob name or <obType>_<platform> "tag" used in GSI naming convention diag_step - the keyword used by GSI to distinguish ges and analysis (i.e. 'ges' or 'anl') date - The cycle date in seconds since epoch is_text_diag - True if using plaintext diag files ''' if is_text_diag: return TEXT_DIAG_PREFIX + obType + "_" + diag_step + "." + epoch_to_yyyymmddHHMM(date) + TEXT_DIAG_SUFFIX else: return DIAG_FILE_PREFIX + obType + "_" + diag_step + "." + epoch_to_yyyymmddHHMM(date) + DIAG_FILE_SUFFIX
def make_plain_text_diag_file(self, outfile='', outdir='', gz=True, include_skipped=False): ''' Make a plain-text diagnostic file given a GSI-generated "binary" diagnostic file PARAMETERS - outfile - Specify an output file name. If none is chosen, use "PLAIN_TEXT_DIAG_PREFIX + "_" + diag_step + "." + epoch_to_yyyymmddHHMM(cycle) + TEXT_DIAG_SUFFIX" - outdir - Specify and output directory. If none is given, use CWD - gz (Default True) - zip the output file using gzip - include_skipped (Default False) - include skipped (i.e. iuse != 1) observations in the output ''' #import pdb ; pdb.set_trace() if len(outdir) == 0: outdir = os.getcwd() if len(outfile) == 0: outfile = TEXT_DIAG_PREFIX + self.ob_type + "_" + self.step + "." + epoch_to_yyyymmddHHMM(self.date) + TEXT_DIAG_SUFFIX if gz: outfile = gzip.open( os.path.join(outdir, outfile), 'wb') else: outfile = open( os.path.join(dataset.get_gsi_output_path(cycle), outfile), 'w') if include_skipped: #all_obs = self.conv_obs + self.rad_obs + self.oz_obs all_obs = self.obs else: #all_obs = self.non_skipped_conv_obs + self.non_skipped_rad_obs + self.non_skipped_oz_obs all_obs = [ ob for ob in self.obs if not ob.skipped ] for ob in all_obs: outfile.write( ob.to_csv() + "\n")
def _extract_gsi_diag(self): ''' Read GSI diag file using a reader, convert entries into GsiObservation objects, and return a list containing these objects ''' #in_file_name = 'diag_' + ob_string + '_' + diag_step + '.' + epoch_to_yyyymmddHHMM(cycle) intermediate_file_name = 'results_' + self.ob_type + '_' + self.step + '.' + epoch_to_yyyymmddHHMM( self.date) # determine ob type log.debug("Processing ob type: %s for step %s" % (self.ob_type, self.step)) # create intermediate text file try: owd = os.getcwd() os.chdir(self.path) except: log.error("Unable to change to directory: %s" % self.path) sys.exit(13) self._create_diag_reader_namelist(self.ob_type, self.diag_file_name, intermediate_file_name) if self.ob_type == 'conv': ret = subprocess.call([self.gsi_diag_reader_conv], shell=True, stdout=open("/dev/null", 'w')) elif self.ob_type == 'oz': raise Exception("Not implemented for oz obs") else: # assume radiance ret = subprocess.call([self.gsi_diag_reader_rad], shell=True, stdout=open("/dev/null", 'w')) # TODO ensure it returns with exit code 9999 AND have option of sending output to a file instead of /dev/null # process output if self.ob_type == 'conv': obs = self.process_conv_diag_reader_output(intermediate_file_name) else: # assume rad obs = self.process_rad_diag_reader_output(intermediate_file_name) try: os.chdir(owd) except: log.error("Unable to switch back to original directory: %s" % owd) sys.exit(13) finally: os.unlink(os.path.join(self.path, intermediate_file_name)) return obs