def clone_wps(self, tgt, vtables, with_files):
"""
Clone the WPS installation directory (self.wps_idir) together with the chosen table files vtables
and additional files with_files. The WPS clone is created in directory tgt.
:param tgt: target directory into which WPS is cloned
:param vtables: a dictionary with keys from list ['geogrid_vtable', 'ungrib_vtable', 'metgrid_vtable'],
which contain paths of the variable tables relative to 'etc/vtables'
:param with_files: a list of files from the WPS source directory that should be symlinked
:return:
"""
src = self.wps_idir
vtable_locs = self.vtable_locations
# build a list of all files that are simply symlinked
symlinks = list(self.wps_exec_files)
symlinks.extend(with_files)
# create target directory (and all intermediate subdirs if necessary)
make_dir(tgt)
# clone all WPS executables
map(lambda x: symlink_unless_exists(osp.join(src, x), osp.join(tgt, x)), symlinks)
# clone all vtables (build symlink name, ensure directories exist, create the symlink)
for vtable_id, vtable_path in vtables.iteritems():
# build path to link location
symlink_path = osp.join(tgt, vtable_locs[vtable_id])
if not osp.exists(symlink_path):
symlink_tgt = osp.join(self.sys_idir, "etc/vtables", vtable_path)
symlink_unless_exists(symlink_tgt, ensure_dir(symlink_path))
def clone_wps(self, tgt, vtables, with_files):
"""
Clone the WPS installation directory (self.wps_idir) together with the chosen table files vtables
and additional files with_files. The WPS clone is created in directory tgt.
:param tgt: target directory into which WPS is cloned
:param vtables: a dictionary with keys from list ['geogrid_vtable', 'ungrib_vtable', 'metgrid_vtable'],
which contain paths of the variable tables relative to 'etc/vtables'
:param with_files: a list of files from the WPS source directory that should be symlinked
:return:
"""
src = self.wps_idir
vtable_locs = self.vtable_locations
# build a list of all files that are simply symlinked
symlinks = list(self.wps_exec_files)
symlinks.extend(with_files)
# create target directory (and all intermediate subdirs if necessary)
make_dir(tgt)
# clone all WPS executables
map(
lambda x: symlink_unless_exists(osp.join(src, x), osp.join(tgt, x)
), symlinks)
# clone all vtables (build symlink name, ensure directories exist, create the symlink)
for vtable_id, vtable_path in vtables.iteritems():
# build path to link location
symlink_path = osp.join(tgt, vtable_locs[vtable_id])
if not osp.exists(symlink_path):
symlink_tgt = osp.join(self.sys_idir, "etc/vtables",
vtable_path)
symlink_unless_exists(symlink_tgt, ensure_dir(symlink_path))
def symlink_gribs(self, manifest, wps_dir):
"""
Make symlinks in the form GRIBFILE.XYZ to all manifest files into wps_dir.
:param manifest: relative paths (w.r.t. ingest_dir) to GRIB files we want linked
:param wps_dir: the WPS directory where we want the symlinks to appear
:return:
"""
for rel_path, grib_name in zip(manifest, generate_grib_names()):
symlink_unless_exists(osp.join(self.ingest_dir, rel_path), osp.join(wps_dir, grib_name))
def symlink_gribs(self, manifest, wps_dir):
"""
Make symlinks in the form GRIBFILE.XYZ to all manifest files into wps_dir.
:param manifest: relative paths (w.r.t. ingest_dir) to GRIB files we want linked
:param wps_dir: the WPS directory where we want the symlinks to appear
:return:
"""
for rel_path, grib_name in zip(manifest, generate_grib_names()):
logging.info('Linking %s -> %s' % ( osp.join(self.ingest_dir, rel_path), osp.join(wps_dir, grib_name)) )
symlink_unless_exists(osp.join(self.ingest_dir, rel_path), osp.join(wps_dir, grib_name))
def clone_vtables(self, tgt):
"""
Clone all vtables (build symlink name, ensure directories exist, create the symlink)
:param tgt: target directory into which WPS is cloned
"""
# where are the symlink locations for vtable files (name of symlink)
vtable_locs = {'geogrid_vtable': 'geogrid/GEOGRID.TBL',
'ungrib_vtable': 'Vtable',
'metgrid_vtable': 'metgrid/METGRID.TBL'}
vtables = self.vtables()
# vtables: a dictionary with keys from list ['geogrid_vtable', 'ungrib_vtable', 'metgrid_vtable'],
# which contain paths of the variable tables relative to 'etc/vtables'
for vtable_id, vtable_path in six.iteritems(vtables):
# build path to link location
symlink_path = osp.join(tgt, vtable_locs[vtable_id])
if not osp.exists(symlink_path):
symlink_tgt = osp.join(self.sys_dir, "etc/vtables", vtable_path)
symlink_unless_exists(symlink_tgt, ensure_dir(symlink_path))
def clone_wrf(self, tgt, with_files):
"""
Clone the WRFV3 directory (self.wrf_idir) into tgt together with the additional files with_files.
:param tgt: target directory into which WRF is cloned
:param with_files: a list of files from the WPS source directory that should be symlinked
:return:
"""
src = osp.join(self.wrf_idir, "run")
# gather all files to symlink in one place
symlinks = list(self.wrf_files)
symlinks.extend(with_files)
if self.wrf_serial_idir is None:
symlinks.extend(self.wrf_serial_files)
# create target directory (and all intermediate subdirs if necessary)
make_dir(tgt)
# symlink all at once
list(
map(
lambda x: symlink_unless_exists(osp.join(src, x),
osp.join(tgt, x)), symlinks))
if self.wrf_serial_idir is not None:
# gather all files to symlink in one place
symlinks = list(self.wrf_serial_files)
src = osp.join(self.wrf_serial_idir, "run")
# symlink all at once
list(
map(
lambda x: symlink_unless_exists(osp.join(src, x),
osp.join(tgt, x)),
symlinks))
def clone_wrf(self, tgt, with_files):
"""
Clone the WRFV3 directory (self.wrf_idir) into tgt together with the additional files with_files.
:param tgt: target directory into which WRF is cloned
:param with_files: a list of files from the WPS source directory that should be symlinked
:return:
"""
src = osp.join(self.wrf_idir, "run")
# gather all files to symlink in one place
symlinks = list(self.wrf_files)
symlinks.extend(with_files)
# create target directory (and all intermediate subdirs if necessary)
make_dir(tgt)
# symlink all at once
map(lambda x: symlink_unless_exists(osp.join(src, x), osp.join(tgt, x)), symlinks)
def clone_wps(self, tgt, with_files):
"""
Clone the WPS installation directory (self.wps_idir) together with the chosen table files vtables
and additional files with_files. The WPS clone is created in directory tgt.
:param tgt: target directory into which WPS is cloned
:param with_files: a list of files from the WPS source directory that should be symlinked
:return:
"""
src = self.wps_idir
# build a list of all files that are simply symlinked
symlinks = list(self.wps_exec_files)
symlinks.extend(with_files)
# create target directory (and all intermediate subdirs if necessary)
make_dir(tgt)
# clone all WPS executables
map(lambda x: symlink_unless_exists(osp.join(src, x), osp.join(tgt, x)), symlinks)
def retrieve_gribs_and_run_ungrib(js, grib_source, q):
"""
This function retrieves required GRIB files and runs ungrib.
It returns either 'SUCCESS' or 'FAILURE' on completion.
:param js: the JobState object containing the forecast configuration
:param grib_source: the GribSource object containing ungrib configuration
:param q: the multiprocessing Queue into which we will send either 'SUCCESS' or 'FAILURE'
"""
wps_dir = osp.abspath(js.wps_dir)
grib_dir = osp.join(wps_dir, grib_source.id)
make_clean_dir(grib_dir)
wps_nml = js.wps_nml
try:
logging.info("retrieving GRIB files from %s" % grib_source.id)
download_whole_cycle = js.get('download_whole_cycle', False)
manifest = grib_source.retrieve_gribs(js.start_utc, js.end_utc,
js.ref_utc, js.cycle_start_utc,
download_whole_cycle)
# logging.info('manifest: ' + str(manifest))
cache_colmet = len(manifest) > 1
have_all_colmet = False
if cache_colmet:
have_all_colmet = len(manifest.colmet_missing) == 0
colmet_dir = osp.join(grib_source.cache_dir,
manifest.colmet_prefix)
logging.info('cache colmet %s, have all colmet %s' %
(cache_colmet, have_all_colmet))
if not have_all_colmet:
# this is also if we do not cache
grib_source.symlink_gribs(manifest.grib_files, grib_dir)
send_email(
js, 'grib2', 'Job %s - %d GRIB2 files downloaded.' %
(js.job_id, len(manifest)))
logging.info("running UNGRIB for %s" % grib_source.id)
logging.info(
"step 4: patch namelist for ungrib end execute ungrib on %s files"
% grib_source.id)
update_namelist(wps_nml, grib_source.namelist_wps_keys())
if cache_colmet:
wps_nml['share']['start_date'] = [
utc_to_esmf(manifest.colmet_files_utc[0])
] * js.num_doms
wps_nml['share']['end_date'] = [
utc_to_esmf(manifest.colmet_files_utc[-1])
] * js.num_doms
# logging.info("namelist.wps for UNGRIB: %s" % json.dumps(wps_nml, indent=4, separators=(',', ': ')))
f90nml.write(wps_nml,
osp.join(grib_dir, 'namelist.wps'),
force=True)
grib_source.clone_vtables(grib_dir)
symlink_unless_exists(osp.join(wps_dir, 'ungrib.exe'),
osp.join(grib_dir, 'ungrib.exe'))
print(grib_dir + ':')
os.system('ls -l %s' % grib_dir)
Ungrib(grib_dir).execute().check_output()
print(grib_dir + ':')
os.system('ls -l %s' % grib_dir)
if cache_colmet:
# move output to cache directory
make_dir(colmet_dir)
for f in manifest.colmet_files:
move(osp.join(grib_dir, f), osp.join(colmet_dir, f))
# now all colmet files should be in the cache
if cache_colmet:
for f in manifest.colmet_files:
symlink_unless_exists(osp.join(colmet_dir, f),
osp.join(wps_dir, f))
else:
# move output
for f in glob.glob(osp.join(grib_dir, grib_source.prefix() + '*')):
move(f, wps_dir)
send_email(js, 'ungrib', 'Job %s - ungrib complete.' % js.job_id)
logging.info('UNGRIB complete for %s' % grib_source.id)
q.put('SUCCESS')
except Exception as e:
logging.error('GRIB2/UNGRIB step failed with exception %s' % repr(e))
traceback.print_exc()
q.put('FAILURE')
