def download_live_ocean(parsed_args, config, *args):
yyyymmdd = parsed_args.run_date.format("YYYYMMDD")
dotted_yyyymmdd = parsed_args.run_date.format("YYYY.MM.DD")
ymd = parsed_args.run_date.format("YYYY-MM-DD")
logger.info(f"downloading Salish Sea western boundary day-averaged LiveOcean file for {ymd}")
process_status_url_tmpl = config["temperature salinity"]["download"]["status file url template"]
process_status_url = process_status_url_tmpl.format(yyyymmdd=dotted_yyyymmdd)
with requests.Session() as session:
try:
_is_file_ready(process_status_url, session)
except RetryError as exc:
logger.error(
f"giving up after {exc.last_attempt.attempt_number} attempts: "
f"{exc.last_attempt.value[1]} for {process_status_url}"
)
raise WorkerError
bc_file_url_tmpl = config["temperature salinity"]["download"]["bc file url template"]
bc_file_url = bc_file_url_tmpl.format(yyyymmdd=dotted_yyyymmdd)
dest_dir = Path(config["temperature salinity"]["download"]["dest dir"], yyyymmdd)
filename = config["temperature salinity"]["download"]["file name"]
grp_name = config["file group"]
lib.mkdir(dest_dir, logger, grp_name=grp_name)
get_web_data(bc_file_url, logger_name=NAME, filepath=dest_dir / filename, session=session)
size = os.stat(dest_dir / filename).st_size
logger.info(f"downloaded {size} bytes from {bc_file_url} to {dest_dir / filename}")
if size == 0:
logger.critical(f"Problem! 0 size file: {dest_dir / filename}")
raise WorkerError
nemo_cmd.api.deflate([dest_dir / filename], 1)
checklist = {ymd: os.fspath(dest_dir / filename)}
return checklist
def download_live_ocean(parsed_args, config, *args):
yyyymmdd = parsed_args.run_date.format('YYYYMMDD')
ymd = parsed_args.run_date.format('YYYY-MM-DD')
logger.info(f'downloading hourly Live Ocean forecast starting on {ymd}')
base_url = config['temperature salinity']['download']['url']
dir_prefix = config['temperature salinity']['download']['directory prefix']
filename_tmpl = config['temperature salinity']['download']['file template']
url = f'{base_url}{dir_prefix}{yyyymmdd}/{filename_tmpl}'
hours = config['temperature salinity']['download']['hours range']
dest_dir = Path(config['temperature salinity']['download']['dest dir'],
yyyymmdd)
grp_name = config['file group']
lib.mkdir(str(dest_dir), logger, grp_name=grp_name)
checklist = {ymd: []}
with requests.Session() as session:
for hr in range(hours[0], hours[1] + 1):
filepath = _get_file(url.format(hh=hr),
filename_tmpl.format(hh=hr), dest_dir,
session)
salishsea_tools.UBC_subdomain.get_UBC_subdomain([str(filepath)])
subdomain_filepath = str(filepath).replace('.nc', '_UBC.nc')
logger.debug(f'extracted UBC sub-domain: {subdomain_filepath}',
extra={'subdomain_filepath': subdomain_filepath})
checklist[ymd].append(subdomain_filepath)
filepath.unlink()
nemo_cmd.api.deflate(dest_dir.glob('*.nc'),
math.floor(multiprocessing.cpu_count() / 2))
return checklist
def _mkdirs(dest_dir_root, date, forecast, grp_name):
lib.mkdir(os.path.join(dest_dir_root, date), logger, grp_name=grp_name)
lib.mkdir(
os.path.join(dest_dir_root, date, forecast),
logger,
grp_name=grp_name,
exist_ok=False,
)
def _move_file(expected_file, grib_forecast_dir, grp_name):
"""
:param :py:class:`pathlib.Path` expected_file:
:param :py:class:`pathlib.Path` grib_forecast_dir:
:param str grp_name:
"""
grib_hour_dir = grib_forecast_dir / expected_file.parent.stem
lib.mkdir(grib_hour_dir, logger, grp_name=grp_name)
shutil.move(os.fspath(expected_file), os.fspath(grib_hour_dir))
logger.debug(f"moved {expected_file} to {grib_hour_dir}/")
def collect_weather(parsed_args, config, *args):
"""
:param :py:class:`argparse.Namespace` parsed_args:
:param :py:class:`nemo_nowcast.Config` config:
:return: Nowcast system checklist items
:rtype: dict
"""
forecast = parsed_args.forecast
resolution = parsed_args.resolution.replace("km", " km")
forecast_yyyymmdd = (
parsed_args.backfill_date.format("YYYYMMDD")
if parsed_args.backfill
else arrow.utcnow().shift(hours=-int(forecast) + 4).format("YYYYMMDD")
)
datamart_dir = Path(config["weather"]["download"][resolution]["datamart dir"])
grib_dir = Path(config["weather"]["download"][resolution]["GRIB dir"])
grp_name = config["file group"]
expected_files = _calc_expected_files(
datamart_dir, forecast, forecast_yyyymmdd, resolution, config
)
lib.mkdir(grib_dir / forecast_yyyymmdd, logger, grp_name=grp_name)
logger.debug(f"created {grib_dir / forecast_yyyymmdd}/")
lib.mkdir(grib_dir / forecast_yyyymmdd / forecast, logger, grp_name=grp_name)
logger.debug(f"created {grib_dir / forecast_yyyymmdd/forecast}/")
if parsed_args.backfill:
logger.info(
f"starting to move {parsed_args.backfill_date.format('YYYY-MM-DD')} files from {datamart_dir / forecast}/"
)
for expected_file in expected_files:
_move_file(expected_file, grib_dir / forecast_yyyymmdd / forecast, grp_name)
else:
handler = _GribFileEventHandler(
expected_files, grib_dir / forecast_yyyymmdd / forecast, grp_name
)
observer = watchdog.observers.Observer()
observer.schedule(handler, os.fspath(datamart_dir / forecast), recursive=True)
logger.info(f"starting to watch for files in {datamart_dir/forecast}/")
observer.start()
while expected_files:
time.sleep(1)
logger.info(
f"finished collecting files from {datamart_dir/forecast}/ to "
f"{grib_dir / forecast_yyyymmdd / forecast}/"
)
checklist = {
f"{forecast} {resolution.replace(' km', 'km')}": os.fspath(
grib_dir / forecast_yyyymmdd / forecast
)
}
return checklist
def _copy_csv_to_results_dir(data_file, run_date, run_type, config):
results_date = run_date if run_type == "nowcast" else run_date.shift(
days=-1)
results_dir = Path(config["results archive"][run_type],
results_date.format("DDMMMYY").lower())
lib.mkdir(results_dir,
logger,
grp_name=config["file group"],
exist_ok=True)
shutil.copy2(data_file, results_dir)
logger.debug(f"copied {data_file} to {results_dir}")
def get_grib(parsed_args, config, *args):
forecast = parsed_args.forecast
resolution = parsed_args.resolution.replace("km", " km")
date = _calc_date(parsed_args, forecast)
logger.info(
f"downloading {forecast} {resolution} forecast GRIB2 files for {date}")
dest_dir_root = config["weather"]["download"][resolution]["GRIB dir"]
grp_name = config["file group"]
_mkdirs(dest_dir_root, date, forecast, grp_name)
url_tmpl = config["weather"]["download"][resolution]["url template"]
filename_tmpl = config["weather"]["download"][resolution]["file template"]
forecast_duration = config["weather"]["download"][resolution][
"forecast duration"]
with requests.Session() as session:
if parsed_args.no_verify_certs:
session.verify = False
for forecast_hour in range(1, forecast_duration + 1):
hr_str = f"{forecast_hour:0=3}"
lib.mkdir(
os.path.join(dest_dir_root, date, forecast, hr_str),
logger,
grp_name=grp_name,
exist_ok=False,
)
for var in config["weather"]["download"][resolution][
"grib variables"]:
filepath = _get_file(
url_tmpl,
filename_tmpl,
var,
dest_dir_root,
date,
forecast,
hr_str,
session,
)
lib.fix_perms(filepath)
checklist = {
f"{forecast} {resolution.replace(' km', 'km')}":
os.path.join(dest_dir_root, date, forecast)
}
return checklist
def get_grib(parsed_args, config, *args):
forecast = parsed_args.forecast
date = _calc_date(parsed_args, forecast)
logger.info(f'downloading {forecast} forecast GRIB2 files for {date}',
extra={'forecast': parsed_args.forecast})
dest_dir_root = config['weather']['GRIB dir']
grp_name = config['file group']
_mkdirs(dest_dir_root, date, forecast, grp_name)
with requests.Session() as session:
for forecast_hour in range(1, FORECAST_DURATION + 1):
hr_str = f'{forecast_hour:0=3}'
lib.mkdir(os.path.join(dest_dir_root, date, forecast, hr_str),
logger,
grp_name=grp_name,
exist_ok=False)
for var in GRIB_VARIABLES:
filepath = _get_file(var, dest_dir_root, date, forecast,
hr_str, session)
os.chmod(filepath, FilePerms(user='******', group='rw', other='r'))
checklist = {f'{date} {forecast} forecast': True}
return checklist
def _render_figures(config, model, run_type, plot_type, dmy, fig_functions,
test_figure_id):
logger.info(
f"starting to render {model} {run_type} {plot_type} {dmy} figures")
checklist = {}
fig_files = []
for svg_name, func in fig_functions.items():
fig_func = func["function"]
args = func.get("args", [])
kwargs = func.get("kwargs", {})
fig_save_format = func.get("format", "svg")
image_loop_figure = func.get("image loop", False)
test_figure = False
if test_figure_id:
test_figure = any((
svg_name == test_figure_id,
image_loop_figure and svg_name.startswith(test_figure_id),
fig_func.__module__.endswith(f"{plot_type}.{test_figure_id}"),
))
if not test_figure:
continue
logger.debug(f"starting {fig_func.__module__}.{fig_func.__name__}")
try:
fig = _calc_figure(fig_func, args, kwargs)
except (FileNotFoundError, IndexError, KeyError, TypeError):
# **IMPORTANT**: the collection of exception above must match those
# handled in the _calc_figure() function
continue
if test_figure:
fig_files_dir = Path(config["figures"]["test path"], run_type, dmy)
fig_files_dir.mkdir(parents=True, exist_ok=True)
else:
fig_files_dir = (Path(config["figures"]["storage path"], run_type,
dmy) if model == "nemo" else Path(
config["figures"]["storage path"], model,
run_type, dmy))
lib.mkdir(fig_files_dir, logger, grp_name=config["file group"])
filename = fig_files_dir / f"{svg_name}_{dmy}.{fig_save_format}"
if image_loop_figure:
filename = fig_files_dir / f"{svg_name}.{fig_save_format}"
fig.savefig(os.fspath(filename),
facecolor=fig.get_facecolor(),
bbox_inches="tight")
logger.debug(f"{filename} saved")
matplotlib.pyplot.close(fig)
if fig_save_format == "svg":
logger.debug(f"starting SVG scouring of {filename}")
tmpfilename = filename.with_suffix(".scour")
scour = Path(os.environ["NOWCAST_ENV"], "bin", "scour")
cmd = f"{scour} {filename} {tmpfilename}"
logger.debug(f"running subprocess: {cmd}")
try:
proc = subprocess.run(
shlex.split(cmd),
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
)
except subprocess.CalledProcessError as e:
logger.warning(
"SVG scouring failed, proceeding with unscoured figure")
logger.debug(f"scour return code: {e.returncode}")
if e.output:
logger.debug(e.output)
continue
logger.debug(proc.stdout)
tmpfilename.rename(filename)
logger.debug(f"{filename} scoured")
lib.fix_perms(filename, grp_name=config["file group"])
fig_files.append(os.fspath(filename))
fig_path = _render_storm_surge_alerts_thumbnail(
config,
run_type,
plot_type,
dmy,
fig,
svg_name,
fig_save_format,
test_figure,
)
if checklist is not None:
checklist["storm surge alerts thumbnail"] = fig_path
checklist[f"{model} {run_type} {plot_type}"] = fig_files
logger.info(
f"finished rendering {model} {run_type} {plot_type} {dmy} figures")
return checklist
def make_surface_current_tiles(parsed_args, config, *args):
"""
:param :py:class:`argparse.Namespace` parsed_args:
:param :py:class:`nemo_nowcast.Config` config:
:return: Nowcast system checklist items
:rtype: dict
"""
run_type = parsed_args.run_type
run_date = parsed_args.run_date
num_procs = parsed_args.nprocs
dmy = run_date.format("DDMMMYY").lower()
dmym1 = run_date.shift(days=-1).format("DDMMMYY").lower()
## TODO: Change to get results from ERDDAP rolling forecast for run_type == forecast*
results_dir0 = (
Path(
config["results archive"]["nowcast-green"],
run_date.shift(days=+2).format("DDMMMYY").lower(),
)
if run_type == "nowcast-green"
else Path(config["results archive"][run_type], dmy)
)
if run_type == "nowcast-green":
results_dirm1 = Path(
config["results archive"]["nowcast-green"],
run_date.shift(days=+1).format("DDMMMYY").lower(),
)
results_dirm2 = Path(
config["results archive"]["nowcast-green"],
run_date.format("DDMMMYY").lower(),
)
if run_type == "forecast":
results_dirm1 = Path(config["results archive"]["nowcast"], dmy)
results_dirm2 = Path(config["results archive"]["nowcast"], dmym1)
if run_type == "forecast2":
results_dirm1 = Path(config["results archive"]["forecast"], dmy)
results_dirm2 = Path(config["results archive"]["nowcast"], dmy)
grid_dir = Path(config["figures"]["grid dir"])
coordf = grid_dir / config["run types"][run_type]["coordinates"]
mesh_maskf = grid_dir / config["run types"][run_type]["mesh mask"]
bathyf = grid_dir / config["run types"][run_type]["bathymetry"]
storage_path = Path(
config["figures"]["surface current tiles"]["storage path"], run_type, dmy
)
lib.mkdir(storage_path, logger, grp_name=config["file group"])
# Loop over last 48h and this forecast{,2}
logger.info(
f"starting rendering of tiles for {run_date.format('YYYY-MM-DD')} {run_type} "
f"in {num_procs} processes into {storage_path}"
)
for results_dir in [results_dirm2, results_dirm1, results_dir0]:
u_list = glob(os.fspath(results_dir) + "/SalishSea_1h_*_grid_U.nc")
v_list = glob(os.fspath(results_dir) + "/SalishSea_1h_*_grid_V.nc")
Uf = Path(u_list[0])
Vf = Path(v_list[0])
with netCDF4.Dataset(Uf) as dsU:
max_time_index = dsU.dimensions["time_counter"].size
units = dsU.variables["time_counter"].units
calendar = dsU.variables["time_counter"].calendar
sec = dsU.variables["time_counter"][:]
expansion_factor = 0.1 # 10% overlap for each tile
logger.debug(f"creating figures using {num_procs} concurrent process(es)")
# Add tasks to a joinable queue
q = multiprocessing.JoinableQueue()
for t_index in range(max_time_index):
task = (
t_index,
sec,
units,
calendar,
run_date,
Uf,
Vf,
coordf,
mesh_maskf,
bathyf,
tile_coords_dic,
expansion_factor,
storage_path,
)
q.put(task)
# Spawn a set of worker processes
procs = []
for i in range(num_procs):
p = multiprocessing.Process(target=_process_time_slice, args=(q,))
procs.append(p)
# Start each one
for p in procs:
p.start()
# Wait until they complete
for p in procs:
p.join()
# Close the queue
q.close()
_pdf_concatenate(storage_path, tile_coords_dic)
checklist = {
run_type: {
"run date": run_date.format("YYYY-MM-DD"),
"png": sorted(
[os.fspath(f) for f in storage_path.iterdir() if f.suffix == ".png"]
),
"pdf": sorted(
[os.fspath(f) for f in storage_path.iterdir() if f.suffix == ".pdf"]
),
}
}
logger.info(
f"finished rendering of tiles for {run_date.format('YYYY-MM-DD')} {run_type} "
f"into {storage_path}"
)
return checklist
def get_NeahBay_ssh(parsed_args, config, *args):
"""Generate sea surface height forcing files from the Neah Bay
storm surge website.
"""
run_type = parsed_args.run_type
with nc.Dataset(config['coordinates']) as coordinates:
lats = coordinates.variables['nav_lat'][:]
lons = coordinates.variables['nav_lon'][:]
logger.debug(f'loaded lats & lons from {config["coordinates"]}',
extra={'run_type': run_type})
# Scrape the surge data from the website into a text file,
# store the file in the run results directory,
# and load the data for processing into netCDF4 files
utc_now = datetime.datetime.now(pytz.timezone('UTC'))
textfile = _read_website(config['ssh']['ssh dir'])
lib.fix_perms(textfile, grp_name=config['file group'])
checklist = {'txt': os.path.basename(textfile)}
# Store a copy of the text file in the run results directory so that
# there is definitive record of the sea surface height data that was
# used for the run
run_date = _utc_now_to_run_date(utc_now, run_type)
results_dir = os.path.join(config['results archive'][run_type],
run_date.strftime('%d%b%y').lower())
lib.mkdir(results_dir,
logger,
grp_name=config['file group'],
exist_ok=True)
shutil.copy2(textfile, results_dir)
# Grab all surge data in the textfile
dates, sshs, fflags = residuals.NeahBay_forcing_anom(
textfile, run_date, config['ssh']['tidal predictions'])
# Identify days with full ssh information
dates_full = _list_full_days(dates, sshs, fflags)
# Set up plotting
fig, ax = _setup_plotting()
# Loop through full days and save netcdf
ip = 0
for d in dates_full:
tc, _, sshd, fflagd = _isolate_day(d, dates, sshs, fflags)
forecast_flag = fflagd.any()
# Plotting
if ip < 3:
ax.plot(sshd, '-o', lw=2, label=d.strftime('%d-%b-%Y'))
ip = ip + 1
filepath = _save_netcdf(d, tc, sshd, forecast_flag, textfile, config,
lats, lons)
filename = os.path.basename(filepath)
lib.fix_perms(filename, grp_name=config['file group'])
if forecast_flag:
if 'fcst' in checklist:
checklist['fcst'].append(filename)
else:
checklist['fcst'] = [filename]
else:
checklist['obs'] = filename
ax.legend(loc=4)
image_file = config['ssh']['monitoring image']
canvas = matplotlib.backends.backend_agg.FigureCanvasAgg(fig)
canvas.print_figure(image_file)
lib.fix_perms(image_file, grp_name=config['file group'])
return checklist
def _render_figures(config, run_type, plot_type, dmy, fig_functions,
test_figure_id):
checklist = {}
fig_files = []
for svg_name, func in fig_functions.items():
fig_func = func['function']
args = func.get('args', [])
kwargs = func.get('kwargs', {})
fig_save_format = func.get('format', 'svg')
test_figure = False
if test_figure_id:
test_figure = any((
svg_name == test_figure_id,
fig_func.__module__.endswith(f'{plot_type}.{test_figure_id}'),
# legacy: for figures.figures module functions
fig_func.__name__ == test_figure_id,
))
if not test_figure:
continue
logger.debug(f'starting {fig_func.__module__}.{fig_func.__name__}')
try:
fig = _calc_figure(fig_func, args, kwargs)
except (FileNotFoundError, IndexError, KeyError, TypeError):
# **IMPORTANT**: the collection of exception above must match those
# handled in the _calc_figure() function
continue
if test_figure:
fig_files_dir = Path(config['figures']['test path'], run_type, dmy)
fig_files_dir.mkdir(parents=True, exist_ok=True)
else:
fig_files_dir = Path(config['figures']['storage path'], run_type,
dmy)
lib.mkdir(os.fspath(fig_files_dir),
logger,
grp_name=config['file group'])
filename = fig_files_dir / f'{svg_name}_{dmy}.{fig_save_format}'
fig.savefig(os.fspath(filename),
facecolor=fig.get_facecolor(),
bbox_inches='tight')
lib.fix_perms(os.fspath(filename), grp_name=config['file group'])
logger.info(f'{filename} saved')
try:
cmd = "scour " + str(filename) + " " + str(filename) + ".scour.svg"
r = subprocess.run(shlex.split(cmd))
if r.returncode == 0:
os.rename(str(filename) + ".scour.svg", str(filename))
lib.fix_perms(os.fspath(filename),
grp_name=config['file group'])
logger.info(f'scoured {filename} saved')
else:
logger.warning(
f'Scouring failed, proceeded with unscoured image')
except (FileNotFoundError, TypeError):
logger.warning('Scouring failed,proceeded with unscoured image.')
fig_files.append(os.fspath(filename))
fig_path = _render_storm_surge_alerts_thumbnail(
config, run_type, plot_type, dmy, fig, svg_name, fig_save_format,
test_figure)
if checklist is not None:
checklist['storm surge alerts thumbnail'] = fig_path
checklist[f'{run_type} {plot_type}'] = fig_files
return checklist
