def clone_wps(self, tgt, vtables, with_files): """ Clone the WPS installation directory (self.wps_idir) together with the chosen table files vtables and additional files with_files. The WPS clone is created in directory tgt. :param tgt: target directory into which WPS is cloned :param vtables: a dictionary with keys from list ['geogrid_vtable', 'ungrib_vtable', 'metgrid_vtable'], which contain paths of the variable tables relative to 'etc/vtables' :param with_files: a list of files from the WPS source directory that should be symlinked :return: """ src = self.wps_idir vtable_locs = self.vtable_locations # build a list of all files that are simply symlinked symlinks = list(self.wps_exec_files) symlinks.extend(with_files) # create target directory (and all intermediate subdirs if necessary) make_dir(tgt) # clone all WPS executables map(lambda x: symlink_unless_exists(osp.join(src, x), osp.join(tgt, x)), symlinks) # clone all vtables (build symlink name, ensure directories exist, create the symlink) for vtable_id, vtable_path in vtables.iteritems(): # build path to link location symlink_path = osp.join(tgt, vtable_locs[vtable_id]) if not osp.exists(symlink_path): symlink_tgt = osp.join(self.sys_idir, "etc/vtables", vtable_path) symlink_unless_exists(symlink_tgt, ensure_dir(symlink_path))
def clone_wps(self, tgt, vtables, with_files): """ Clone the WPS installation directory (self.wps_idir) together with the chosen table files vtables and additional files with_files. The WPS clone is created in directory tgt. :param tgt: target directory into which WPS is cloned :param vtables: a dictionary with keys from list ['geogrid_vtable', 'ungrib_vtable', 'metgrid_vtable'], which contain paths of the variable tables relative to 'etc/vtables' :param with_files: a list of files from the WPS source directory that should be symlinked :return: """ src = self.wps_idir vtable_locs = self.vtable_locations # build a list of all files that are simply symlinked symlinks = list(self.wps_exec_files) symlinks.extend(with_files) # create target directory (and all intermediate subdirs if necessary) make_dir(tgt) # clone all WPS executables map( lambda x: symlink_unless_exists(osp.join(src, x), osp.join(tgt, x) ), symlinks) # clone all vtables (build symlink name, ensure directories exist, create the symlink) for vtable_id, vtable_path in vtables.iteritems(): # build path to link location symlink_path = osp.join(tgt, vtable_locs[vtable_id]) if not osp.exists(symlink_path): symlink_tgt = osp.join(self.sys_idir, "etc/vtables", vtable_path) symlink_unless_exists(symlink_tgt, ensure_dir(symlink_path))
def symlink_gribs(self, manifest, wps_dir): """ Make symlinks in the form GRIBFILE.XYZ to all manifest files into wps_dir. :param manifest: relative paths (w.r.t. ingest_dir) to GRIB files we want linked :param wps_dir: the WPS directory where we want the symlinks to appear :return: """ for rel_path, grib_name in zip(manifest, generate_grib_names()): symlink_unless_exists(osp.join(self.ingest_dir, rel_path), osp.join(wps_dir, grib_name))
def symlink_gribs(self, manifest, wps_dir): """ Make symlinks in the form GRIBFILE.XYZ to all manifest files into wps_dir. :param manifest: relative paths (w.r.t. ingest_dir) to GRIB files we want linked :param wps_dir: the WPS directory where we want the symlinks to appear :return: """ for rel_path, grib_name in zip(manifest, generate_grib_names()): logging.info('Linking %s -> %s' % ( osp.join(self.ingest_dir, rel_path), osp.join(wps_dir, grib_name)) ) symlink_unless_exists(osp.join(self.ingest_dir, rel_path), osp.join(wps_dir, grib_name))
def clone_vtables(self, tgt): """ Clone all vtables (build symlink name, ensure directories exist, create the symlink) :param tgt: target directory into which WPS is cloned """ # where are the symlink locations for vtable files (name of symlink) vtable_locs = {'geogrid_vtable': 'geogrid/GEOGRID.TBL', 'ungrib_vtable': 'Vtable', 'metgrid_vtable': 'metgrid/METGRID.TBL'} vtables = self.vtables() # vtables: a dictionary with keys from list ['geogrid_vtable', 'ungrib_vtable', 'metgrid_vtable'], # which contain paths of the variable tables relative to 'etc/vtables' for vtable_id, vtable_path in six.iteritems(vtables): # build path to link location symlink_path = osp.join(tgt, vtable_locs[vtable_id]) if not osp.exists(symlink_path): symlink_tgt = osp.join(self.sys_dir, "etc/vtables", vtable_path) symlink_unless_exists(symlink_tgt, ensure_dir(symlink_path))
def clone_wrf(self, tgt, with_files): """ Clone the WRFV3 directory (self.wrf_idir) into tgt together with the additional files with_files. :param tgt: target directory into which WRF is cloned :param with_files: a list of files from the WPS source directory that should be symlinked :return: """ src = osp.join(self.wrf_idir, "run") # gather all files to symlink in one place symlinks = list(self.wrf_files) symlinks.extend(with_files) if self.wrf_serial_idir is None: symlinks.extend(self.wrf_serial_files) # create target directory (and all intermediate subdirs if necessary) make_dir(tgt) # symlink all at once list( map( lambda x: symlink_unless_exists(osp.join(src, x), osp.join(tgt, x)), symlinks)) if self.wrf_serial_idir is not None: # gather all files to symlink in one place symlinks = list(self.wrf_serial_files) src = osp.join(self.wrf_serial_idir, "run") # symlink all at once list( map( lambda x: symlink_unless_exists(osp.join(src, x), osp.join(tgt, x)), symlinks))
def clone_wrf(self, tgt, with_files): """ Clone the WRFV3 directory (self.wrf_idir) into tgt together with the additional files with_files. :param tgt: target directory into which WRF is cloned :param with_files: a list of files from the WPS source directory that should be symlinked :return: """ src = osp.join(self.wrf_idir, "run") # gather all files to symlink in one place symlinks = list(self.wrf_files) symlinks.extend(with_files) # create target directory (and all intermediate subdirs if necessary) make_dir(tgt) # symlink all at once map(lambda x: symlink_unless_exists(osp.join(src, x), osp.join(tgt, x)), symlinks)
def clone_wps(self, tgt, with_files): """ Clone the WPS installation directory (self.wps_idir) together with the chosen table files vtables and additional files with_files. The WPS clone is created in directory tgt. :param tgt: target directory into which WPS is cloned :param with_files: a list of files from the WPS source directory that should be symlinked :return: """ src = self.wps_idir # build a list of all files that are simply symlinked symlinks = list(self.wps_exec_files) symlinks.extend(with_files) # create target directory (and all intermediate subdirs if necessary) make_dir(tgt) # clone all WPS executables map(lambda x: symlink_unless_exists(osp.join(src, x), osp.join(tgt, x)), symlinks)
def retrieve_gribs_and_run_ungrib(js, grib_source, q): """ This function retrieves required GRIB files and runs ungrib. It returns either 'SUCCESS' or 'FAILURE' on completion. :param js: the JobState object containing the forecast configuration :param grib_source: the GribSource object containing ungrib configuration :param q: the multiprocessing Queue into which we will send either 'SUCCESS' or 'FAILURE' """ wps_dir = osp.abspath(js.wps_dir) grib_dir = osp.join(wps_dir, grib_source.id) make_clean_dir(grib_dir) wps_nml = js.wps_nml try: logging.info("retrieving GRIB files from %s" % grib_source.id) download_whole_cycle = js.get('download_whole_cycle', False) manifest = grib_source.retrieve_gribs(js.start_utc, js.end_utc, js.ref_utc, js.cycle_start_utc, download_whole_cycle) # logging.info('manifest: ' + str(manifest)) cache_colmet = len(manifest) > 1 have_all_colmet = False if cache_colmet: have_all_colmet = len(manifest.colmet_missing) == 0 colmet_dir = osp.join(grib_source.cache_dir, manifest.colmet_prefix) logging.info('cache colmet %s, have all colmet %s' % (cache_colmet, have_all_colmet)) if not have_all_colmet: # this is also if we do not cache grib_source.symlink_gribs(manifest.grib_files, grib_dir) send_email( js, 'grib2', 'Job %s - %d GRIB2 files downloaded.' % (js.job_id, len(manifest))) logging.info("running UNGRIB for %s" % grib_source.id) logging.info( "step 4: patch namelist for ungrib end execute ungrib on %s files" % grib_source.id) update_namelist(wps_nml, grib_source.namelist_wps_keys()) if cache_colmet: wps_nml['share']['start_date'] = [ utc_to_esmf(manifest.colmet_files_utc[0]) ] * js.num_doms wps_nml['share']['end_date'] = [ utc_to_esmf(manifest.colmet_files_utc[-1]) ] * js.num_doms # logging.info("namelist.wps for UNGRIB: %s" % json.dumps(wps_nml, indent=4, separators=(',', ': '))) f90nml.write(wps_nml, osp.join(grib_dir, 'namelist.wps'), force=True) grib_source.clone_vtables(grib_dir) symlink_unless_exists(osp.join(wps_dir, 'ungrib.exe'), osp.join(grib_dir, 'ungrib.exe')) print(grib_dir + ':') os.system('ls -l %s' % grib_dir) Ungrib(grib_dir).execute().check_output() print(grib_dir + ':') os.system('ls -l %s' % grib_dir) if cache_colmet: # move output to cache directory make_dir(colmet_dir) for f in manifest.colmet_files: move(osp.join(grib_dir, f), osp.join(colmet_dir, f)) # now all colmet files should be in the cache if cache_colmet: for f in manifest.colmet_files: symlink_unless_exists(osp.join(colmet_dir, f), osp.join(wps_dir, f)) else: # move output for f in glob.glob(osp.join(grib_dir, grib_source.prefix() + '*')): move(f, wps_dir) send_email(js, 'ungrib', 'Job %s - ungrib complete.' % js.job_id) logging.info('UNGRIB complete for %s' % grib_source.id) q.put('SUCCESS') except Exception as e: logging.error('GRIB2/UNGRIB step failed with exception %s' % repr(e)) traceback.print_exc() q.put('FAILURE')