def _crop_to_watersheds(
config,
ymd,
ist,
ien,
jst,
jen,
outgrib,
outzeros,
):
"""Crop the grid to the sub-region of GEM 2.5km operational forecast
grid that encloses the watersheds that are used to calculate river
flows for runoff forcing files for the Salish Sea NEMO model.
"""
OPERdir = config['weather']['ops dir']
wgrib2 = config['weather']['wgrib2']
newgrib = os.path.join(OPERdir,
'oper_allvar_small_{ymd}.grib'.format(ymd=ymd))
newzeros = os.path.join(OPERdir,
'oper_000_small_{ymd}.grib'.format(ymd=ymd))
istr = '{ist}:{ien}'.format(ist=ist, ien=ien)
jstr = '{jst}:{jen}'.format(jst=jst, jen=jen)
cmd = [wgrib2, outgrib, '-ijsmall_grib', istr, jstr, newgrib]
lib.run_in_subprocess(cmd, wgrib2_logger.debug, logger.error)
logger.debug(
'cropped hourly file to watersheds sub-region: {}'.format(newgrib))
cmd = [wgrib2, outzeros, '-ijsmall_grib', istr, jstr, newzeros]
lib.run_in_subprocess(cmd, wgrib2_logger.debug, logger.error)
logger.debug(
'cropped zero-hour file to watersheds sub-region: {}'.format(newgrib))
os.remove(outgrib)
os.remove(outzeros)
return newgrib, newzeros
def _rotate_grib_wind(config, fcst_section_hrs):
"""Use wgrib2 to consolidate each hour's u and v wind components into a
single file and then rotate the wind direction to geographical
coordinates.
"""
GRIBdir = config['weather']['GRIB dir']
wgrib2 = config['weather']['wgrib2']
grid_defn = config['weather']['grid_defn.pl']
# grid_defn.pl expects to find wgrib2 in the pwd,
# create a symbolic link to keep it happy (if its not already there)
try:
os.symlink(wgrib2, 'wgrib2')
except:
pass
for day_fcst, realstart, start_hr, end_hr in fcst_section_hrs.values():
for fhour in range(start_hr, end_hr + 1):
# Set up directories and files
sfhour = '{:03d}'.format(fhour)
suff = '_pid_' + str(os.getpid())
outuv = os.path.join(GRIBdir, day_fcst, sfhour, 'UV.grib' + suff)
outuvrot = os.path.join(GRIBdir, day_fcst, sfhour,
'UVrot.grib' + suff)
# Delete residual instances of files that are created so that
# function can be re-run cleanly
try:
os.remove(outuv)
except OSError:
pass
try:
os.remove(outuvrot)
except OSError:
pass
# Consolidate u and v wind component values into one file
for fpattern in ['*UGRD*', '*VGRD*']:
pattern = os.path.join(GRIBdir, day_fcst, sfhour, fpattern)
fn = glob.glob(pattern)
try:
if os.stat(fn[0]).st_size == 0:
logger.critical('Problem: 0 size file {}'.format(
fn[0]))
raise WorkerError
except IndexError:
logger.critical(
'No GRIB files match pattern; '
'a previous download may have failed: {}'.format(
pattern))
raise WorkerError
cmd = [wgrib2, fn[0], '-append', '-grib', outuv]
lib.run_in_subprocess(cmd, wgrib2_logger.debug, logger.error)
# rotate
GRIDspec = subprocess.check_output([grid_defn, outuv])
cmd = [wgrib2, outuv]
cmd.extend('-new_grid_winds earth'.split())
cmd.append('-new_grid')
cmd.extend(GRIDspec.split())
cmd.append(outuvrot)
lib.run_in_subprocess(cmd, wgrib2_logger.debug, logger.error)
os.remove(outuv)
logger.debug('consolidated and rotated wind components')
def _netCDF4_deflate(outnetcdf):
"""Run ncks in a subprocess to convert outnetcdf to netCDF4 format
with it variables compressed with Lempel-Ziv deflation.
"""
cmd = ["ncks", "-4", "-L4", "-O", outnetcdf, outnetcdf]
try:
lib.run_in_subprocess(cmd, logger.debug, logger.error)
logger.debug(f"netCDF4 deflated {outnetcdf}")
except WorkerError:
raise
def _netCDF4_deflate(outnetcdf):
"""Run ncks in a subprocess to convert outnetcdf to netCDF4 format
with it variables compressed with Lempel-Ziv deflation.
"""
cmd = ['ncks', '-4', '-L4', '-O', outnetcdf, outnetcdf]
try:
lib.run_in_subprocess(cmd, logger.debug, logger.error)
logger.debug('netCDF4 deflated {}'.format(outnetcdf))
except WorkerError:
raise
def _rotate_grib_wind(config, fcst_section_hrs):
"""Use wgrib2 to consolidate each hour's u and v wind components into a
single file and then rotate the wind direction to geographical
coordinates.
"""
GRIBdir = config["weather"]["download"]["2.5 km"]["GRIB dir"]
wgrib2 = config["weather"]["wgrib2"]
grid_defn = config["weather"]["grid_defn.pl"]
for day_fcst, realstart, start_hr, end_hr in fcst_section_hrs.values():
for fhour in range(start_hr, end_hr + 1):
# Set up directories and files
sfhour = f"{fhour:03d}"
outuv = os.path.join(GRIBdir, day_fcst, sfhour, "UV.grib")
outuvrot = os.path.join(GRIBdir, day_fcst, sfhour, "UVrot.grib")
# Delete residual instances of files that are created so that
# function can be re-run cleanly
try:
os.remove(outuv)
except OSError:
pass
try:
os.remove(outuvrot)
except OSError:
pass
# Consolidate u and v wind component values into one file
for fpattern in ["*UGRD*", "*VGRD*"]:
pattern = os.path.join(GRIBdir, day_fcst, sfhour, fpattern)
fn = glob.glob(pattern)
try:
if os.stat(fn[0]).st_size == 0:
logger.critical(f"Problem: 0 size file {fn[0]}")
raise WorkerError
except IndexError:
logger.critical(
f"No GRIB files match pattern; a previous download;"
f" may have failed: {pattern}")
raise WorkerError
cmd = [wgrib2, fn[0], "-append", "-grib", outuv]
lib.run_in_subprocess(cmd, wgrib2_logger.debug, logger.error)
# rotate
GRIDspec = subprocess.check_output([grid_defn, outuv],
cwd=os.path.dirname(wgrib2))
cmd = [wgrib2, outuv]
cmd.extend("-new_grid_winds earth".split())
cmd.append("-new_grid")
cmd.extend(GRIDspec.split())
cmd.append(outuvrot)
lib.run_in_subprocess(cmd, wgrib2_logger.debug, logger.error)
os.remove(outuv)
logger.debug("consolidated and rotated wind components")
def _collect_grib_scalars(config, fcst_section_hrs):
"""Use wgrib2 and grid_defn.pl to consolidate each hour's scalar
variables into an single file and then re-grid them to match the
u and v wind components.
"""
GRIBdir = config['weather']['GRIB dir']
wgrib2 = config['weather']['wgrib2']
grid_defn = config['weather']['grid_defn.pl']
# grid_defn.pl expects to find wgrib2 in the pwd,
# create a symbolic link to keep it happy
try:
os.symlink(wgrib2, 'wgrib2')
except:
pass
for day_fcst, realstart, start_hr, end_hr in fcst_section_hrs.values():
for fhour in range(start_hr, end_hr + 1):
# Set up directories and files
sfhour = '{:03d}'.format(fhour)
suff = '_pid_' + str(os.getpid())
outscalar = os.path.join(GRIBdir, day_fcst, sfhour,
'scalar.grib' + suff)
outscalargrid = os.path.join(GRIBdir, day_fcst, sfhour,
'gscalar.grib' + suff)
# Delete residual instances of files that are created so that
# function can be re-run cleanly
try:
os.remove(outscalar)
except OSError:
pass
try:
os.remove(outscalargrid)
except OSError:
pass
# Consolidate scalar variables into one file
for fn in glob.glob(os.path.join(GRIBdir, day_fcst, sfhour, '*')):
if not ('GRD' in fn) and ('CMC' in fn):
cmd = [wgrib2, fn, '-append', '-grib', outscalar]
lib.run_in_subprocess(cmd, wgrib2_logger.debug,
logger.error)
# Re-grid
GRIDspec = subprocess.check_output([grid_defn, outscalar])
cmd = [wgrib2, outscalar]
cmd.append('-new_grid')
cmd.extend(GRIDspec.split())
cmd.append(outscalargrid)
lib.run_in_subprocess(cmd, wgrib2_logger.debug, logger.error)
os.remove(outscalar)
logger.debug('consolidated and re-gridded scalar variables')
def _make_netCDF_files(config, ymd, subdir, outgrib, outzeros):
"""Convert the GRIB files to netcdf (classic) files."""
OPERdir = config["weather"]["ops dir"]
wgrib2 = config["weather"]["wgrib2"]
outnetcdf = os.path.join(OPERdir, subdir, f"ops_{ymd}.nc")
out0netcdf = os.path.join(OPERdir, subdir, f"oper_000_{ymd}.nc")
cmd = [wgrib2, outgrib, "-netcdf", outnetcdf]
lib.run_in_subprocess(cmd, wgrib2_logger.debug, logger.error)
logger.debug(f"created hourly netCDF classic file: {outnetcdf}")
lib.fix_perms(outnetcdf, grp_name=config["file group"])
cmd = [wgrib2, outzeros, "-netcdf", out0netcdf]
lib.run_in_subprocess(cmd, wgrib2_logger.debug, logger.error)
logger.debug(f"created zero-hour netCDF classic file: {out0netcdf}")
os.remove(outgrib)
os.remove(outzeros)
return outnetcdf, out0netcdf
def _make_netCDF_files(config, ymd, subdir, outgrib, outzeros):
"""Convert the GRIB files to netcdf (classic) files.
"""
OPERdir = config['weather']['ops dir']
wgrib2 = config['weather']['wgrib2']
outnetcdf = os.path.join(OPERdir, subdir, f'ops_{ymd}.nc')
out0netcdf = os.path.join(OPERdir, subdir, f'oper_000_{ymd}.nc')
cmd = [wgrib2, outgrib, '-netcdf', outnetcdf]
lib.run_in_subprocess(cmd, wgrib2_logger.debug, logger.error)
logger.debug(f'created hourly netCDF classic file: {outnetcdf}')
lib.fix_perms(outnetcdf, grp_name=config['file group'])
cmd = [wgrib2, outzeros, '-netcdf', out0netcdf]
lib.run_in_subprocess(cmd, wgrib2_logger.debug, logger.error)
logger.debug(f'created zero-hour netCDF classic file: {out0netcdf}')
os.remove(outgrib)
os.remove(outzeros)
return outnetcdf, out0netcdf
def _crop_to_watersheds(config, ymd, ist, ien, jst, jen, outgrib, outzeros):
"""Crop the grid to the sub-region of GEM 2.5km operational forecast
grid that encloses the watersheds that are used to calculate river
flows for runoff forcing files for the Salish Sea NEMO model.
"""
OPERdir = config["weather"]["ops dir"]
wgrib2 = config["weather"]["wgrib2"]
newgrib = os.path.join(OPERdir, f"oper_allvar_small_{ymd}.grib")
newzeros = os.path.join(OPERdir, f"oper_000_small_{ymd}.grib")
istr = f"{ist}:{ien}"
jstr = f"{jst}:{jen}"
cmd = [wgrib2, outgrib, "-ijsmall_grib", istr, jstr, newgrib]
lib.run_in_subprocess(cmd, wgrib2_logger.debug, logger.error)
logger.debug(f"cropped hourly file to watersheds sub-region: {newgrib}")
cmd = [wgrib2, outzeros, "-ijsmall_grib", istr, jstr, newzeros]
lib.run_in_subprocess(cmd, wgrib2_logger.debug, logger.error)
logger.debug(f"cropped zero-hour file to watersheds sub-region: {newgrib}")
os.remove(outgrib)
os.remove(outzeros)
return newgrib, newzeros
def _collect_grib_scalars(config, fcst_section_hrs):
"""Use wgrib2 and grid_defn.pl to consolidate each hour's scalar
variables into an single file and then re-grid them to match the
u and v wind components.
"""
GRIBdir = config["weather"]["download"]["2.5 km"]["GRIB dir"]
wgrib2 = config["weather"]["wgrib2"]
grid_defn = config["weather"]["grid_defn.pl"]
for day_fcst, realstart, start_hr, end_hr in fcst_section_hrs.values():
for fhour in range(start_hr, end_hr + 1):
# Set up directories and files
sfhour = f"{fhour:03d}"
outscalar = os.path.join(GRIBdir, day_fcst, sfhour, "scalar.grib")
outscalargrid = os.path.join(GRIBdir, day_fcst, sfhour,
"gscalar.grib")
# Delete residual instances of files that are created so that
# function can be re-run cleanly
try:
os.remove(outscalar)
except OSError:
pass
try:
os.remove(outscalargrid)
except OSError:
pass
# Consolidate scalar variables into one file
for fn in glob.glob(os.path.join(GRIBdir, day_fcst, sfhour, "*")):
if not ("GRD" in fn) and ("CMC" in fn):
cmd = [wgrib2, fn, "-append", "-grib", outscalar]
lib.run_in_subprocess(cmd, wgrib2_logger.debug,
logger.error)
# Re-grid
GRIDspec = subprocess.check_output([grid_defn, outscalar],
cwd=os.path.dirname(wgrib2))
cmd = [wgrib2, outscalar]
cmd.append("-new_grid")
cmd.extend(GRIDspec.split())
cmd.append(outscalargrid)
lib.run_in_subprocess(cmd, wgrib2_logger.debug, logger.error)
os.remove(outscalar)
logger.debug("consolidated and re-gridded scalar variables")
def _concat_hourly_gribs(config, ymd, fcst_section_hrs):
"""Concatenate in hour order the wind velocity components
and scalar variables from hourly files into a daily file.
Also create the zero-hour file that is used to initialize the
calculation of instantaneous values from the forecast accumulated
values.
"""
GRIBdir = config["weather"]["download"]["2.5 km"]["GRIB dir"]
OPERdir = config["weather"]["ops dir"]
wgrib2 = config["weather"]["wgrib2"]
outgrib = os.path.join(OPERdir, f"oper_allvar_{ymd}.grib")
outzeros = os.path.join(OPERdir, f"oper_000_{ymd}.grib")
# Delete residual instances of files that are created so that
# function can be re-run cleanly
try:
os.remove(outgrib)
except OSError:
pass
try:
os.remove(outzeros)
except OSError:
pass
for day_fcst, realstart, start_hr, end_hr in fcst_section_hrs.values():
for fhour in range(start_hr, end_hr + 1):
# Set up directories and files
sfhour = f"{fhour:03d}"
outuvrot = os.path.join(GRIBdir, day_fcst, sfhour, "UVrot.grib")
outscalargrid = os.path.join(GRIBdir, day_fcst, sfhour,
"gscalar.grib")
if fhour == start_hr and realstart == -1:
cmd = [wgrib2, outuvrot, "-append", "-grib", outzeros]
lib.run_in_subprocess(cmd, wgrib2_logger.debug, logger.error)
cmd = [wgrib2, outscalargrid, "-append", "-grib", outzeros]
lib.run_in_subprocess(cmd, wgrib2_logger.debug, logger.error)
else:
cmd = [wgrib2, outuvrot, "-append", "-grib", outgrib]
lib.run_in_subprocess(cmd, wgrib2_logger.debug, logger.error)
cmd = [wgrib2, outscalargrid, "-append", "-grib", outgrib]
lib.run_in_subprocess(cmd, wgrib2_logger.debug, logger.error)
os.remove(outuvrot)
os.remove(outscalargrid)
logger.debug(
f"concatenated variables in hour order from hourly files to daily "
f"file {outgrib}")
logger.debug(
f"created zero-hour file for initialization of accumulated -> "
f"instantaneous values calculations: {outzeros}")
return outgrib, outzeros
def _concat_hourly_gribs(config, ymd, fcst_section_hrs):
"""Concatenate in hour order the wind velocity components
and scalar variables from hourly files into a daily file.
Also create the zero-hour file that is used to initialize the
calculation of instantaneous values from the forecast accumulated
values.
"""
GRIBdir = config['weather']['GRIB dir']
OPERdir = config['weather']['ops dir']
wgrib2 = config['weather']['wgrib2']
outgrib = os.path.join(OPERdir, 'oper_allvar_{ymd}.grib'.format(ymd=ymd))
outzeros = os.path.join(OPERdir, 'oper_000_{ymd}.grib'.format(ymd=ymd))
# Delete residual instances of files that are created so that
# function can be re-run cleanly
try:
os.remove(outgrib)
except OSError:
pass
try:
os.remove(outzeros)
except OSError:
pass
for day_fcst, realstart, start_hr, end_hr in fcst_section_hrs.values():
for fhour in range(start_hr, end_hr + 1):
# Set up directories and files
sfhour = '{:03d}'.format(fhour)
suff = '_pid_' + str(os.getpid())
outuvrot = os.path.join(GRIBdir, day_fcst, sfhour,
'UVrot.grib' + suff)
outscalargrid = os.path.join(GRIBdir, day_fcst, sfhour,
'gscalar.grib' + suff)
if (fhour == start_hr and realstart == -1):
cmd = [wgrib2, outuvrot, '-append', '-grib', outzeros]
lib.run_in_subprocess(cmd, wgrib2_logger.debug, logger.error)
cmd = [wgrib2, outscalargrid, '-append', '-grib', outzeros]
lib.run_in_subprocess(cmd, wgrib2_logger.debug, logger.error)
else:
cmd = [wgrib2, outuvrot, '-append', '-grib', outgrib]
lib.run_in_subprocess(cmd, wgrib2_logger.debug, logger.error)
cmd = [wgrib2, outscalargrid, '-append', '-grib', outgrib]
lib.run_in_subprocess(cmd, wgrib2_logger.debug, logger.error)
os.remove(outuvrot)
os.remove(outscalargrid)
logger.debug('concatenated variables in hour order from hourly files '
'to daily file {}'.format(outgrib))
logger.debug('created zero-hour file for initialization of accumulated -> '
'instantaneous values calculations: {}'.format(outzeros))
return outgrib, outzeros
def download_results(parsed_args, config, *args):
host_name = parsed_args.host_name
run_type = parsed_args.run_type
dest_host = parsed_args.dest_host
run_date = parsed_args.run_date
try:
try:
# Hindcast special case 1st due to hindcast host in enabled hosts
# with empty run types collection to enable forcing uploads
host_config = config["run"]["hindcast hosts"][host_name]
except KeyError:
host_config = config["run"]["enabled hosts"][host_name]
except KeyError:
logger.critical(f"unrecognized host: {host_name}")
raise WorkerError
results_dir = run_date.format("DDMMMYY").lower()
run_type_results = Path(host_config["run types"][run_type]["results"])
src_dir = run_type_results / results_dir
src = f"{host_name}:{src_dir}"
try:
dest = Path(config["results archive"][run_type])
except TypeError:
dest_path = Path(config["results archive"][run_type][dest_host])
dest = dest_path if dest_host == "localhost" else f"{dest_host}:{dest_path}"
logger.info(f"downloading results from {src} to {dest}")
cmd = shlex.split(f"scp -pr {src} {dest}")
lib.run_in_subprocess(cmd, logger.debug, logger.error)
checklist = {run_type: {"run date": run_date.format("YYYY-MM-DD")}}
if dest_host == "localhost":
results_archive_dir = _tidy_localhost(run_type, dest, results_dir,
config)
for freq in "1h 1d".split():
checklist[run_type][freq] = list(
map(os.fspath,
results_archive_dir.glob(f"*SalishSea_{freq}_*.nc")))
else:
_tidy_dest_host(run_type, dest_host, dest_path, results_dir, config)
checklist[run_type]["destination"] = dest
return checklist
def download_wwatch3_results(parsed_args, config, *args):
host_name = parsed_args.host_name
run_type = parsed_args.run_type
run_date = parsed_args.run_date
results_dir = run_date.format("DDMMMYY").lower()
run_type_results = Path(config["wave forecasts"]["results"][run_type])
src = f"{host_name}:{run_type_results / results_dir}"
dest = Path(config["wave forecasts"]["results archive"][run_type])
cmd = shlex.split(f"scp -Cpr {src} {dest}")
lib.run_in_subprocess(cmd, logger.debug, logger.error)
results_archive_dir = dest / results_dir
lib.fix_perms(
dest / results_dir,
mode=int(lib.FilePerms(user="******", group="rwx", other="rx")),
grp_name=config["file group"],
)
for filepath in results_archive_dir.glob("*"):
lib.fix_perms(filepath, grp_name=config["file group"])
checklist = {
run_type:
list(map(os.fspath, results_archive_dir.glob(f"SoG_ww3_*.nc")))
}
return checklist
def download_results(parsed_args, config, *args):
host_name = parsed_args.host_name
run_date = parsed_args.run_date
run_type = parsed_args.run_type
host_run_config = config['run'][host_name]
results_dir = run_date.strftime('%d%b%y').lower()
run_type_results = Path(host_run_config['results'][run_type])
src_dir = run_type_results / results_dir
src = f'{host_name}:{src_dir}'
dest = Path(config['results archive'][run_type])
cmd = ['scp', '-Cpr', src, str(dest)]
lib.run_in_subprocess(cmd, logger.debug, logger.error)
lib.fix_perms(str(dest / results_dir),
mode=lib.PERMS_RWX_RWX_R_X,
grp_name='sallen')
results_archive_dir = dest / results_dir
for filepath in results_archive_dir.glob('*'):
lib.fix_perms(str(filepath), grp_name='sallen')
checklist = {run_type: {}}
for freq in '1h 1d'.split():
checklist[run_type][freq] = list(
map(str, results_archive_dir.glob(f'SalishSea_{freq}_*.nc')))
return checklist
def download_fvcom_results(parsed_args, config, *args):
"""
:param :py:class:`argparse.Namespace` parsed_args:
:param :py:class:`nemo_nowcast.Config` config:
:return: Nowcast system checklist items
:rtype: dict
"""
host_name = parsed_args.host_name
model_config = parsed_args.model_config
run_type = parsed_args.run_type
run_date = parsed_args.run_date
results_dir = run_date.format("DDMMMYY").lower()
run_type_results = Path(config["vhfr fvcom runs"]["run types"]
[f"{run_type} {model_config}"]["results"])
src = f"{host_name}:{run_type_results / results_dir}"
dest = Path(config["vhfr fvcom runs"]["results archive"]
[f"{run_type} {model_config}"])
cmd = shlex.split(f"scp -Cpr {src} {dest}")
lib.run_in_subprocess(cmd, logger.debug, logger.error)
results_archive_dir = dest / results_dir
lib.fix_perms(
dest / results_dir,
mode=int(lib.FilePerms(user="******", group="rwx", other="rx")),
grp_name=config["file group"],
)
for filepath in results_archive_dir.glob("*"):
lib.fix_perms(filepath, grp_name=config["file group"])
checklist = {
run_type: {
"host": host_name,
"model config": model_config,
"run date": run_date.format("YYYY-MM-DD"),
"files": list(map(os.fspath, results_archive_dir.glob("vh*.nc"))),
}
}
return checklist