def download_live_ocean(parsed_args, config, *args): yyyymmdd = parsed_args.run_date.format("YYYYMMDD") dotted_yyyymmdd = parsed_args.run_date.format("YYYY.MM.DD") ymd = parsed_args.run_date.format("YYYY-MM-DD") logger.info(f"downloading Salish Sea western boundary day-averaged LiveOcean file for {ymd}") process_status_url_tmpl = config["temperature salinity"]["download"]["status file url template"] process_status_url = process_status_url_tmpl.format(yyyymmdd=dotted_yyyymmdd) with requests.Session() as session: try: _is_file_ready(process_status_url, session) except RetryError as exc: logger.error( f"giving up after {exc.last_attempt.attempt_number} attempts: " f"{exc.last_attempt.value[1]} for {process_status_url}" ) raise WorkerError bc_file_url_tmpl = config["temperature salinity"]["download"]["bc file url template"] bc_file_url = bc_file_url_tmpl.format(yyyymmdd=dotted_yyyymmdd) dest_dir = Path(config["temperature salinity"]["download"]["dest dir"], yyyymmdd) filename = config["temperature salinity"]["download"]["file name"] grp_name = config["file group"] lib.mkdir(dest_dir, logger, grp_name=grp_name) get_web_data(bc_file_url, logger_name=NAME, filepath=dest_dir / filename, session=session) size = os.stat(dest_dir / filename).st_size logger.info(f"downloaded {size} bytes from {bc_file_url} to {dest_dir / filename}") if size == 0: logger.critical(f"Problem! 0 size file: {dest_dir / filename}") raise WorkerError nemo_cmd.api.deflate([dest_dir / filename], 1) checklist = {ymd: os.fspath(dest_dir / filename)} return checklist
def download_live_ocean(parsed_args, config, *args): yyyymmdd = parsed_args.run_date.format('YYYYMMDD') ymd = parsed_args.run_date.format('YYYY-MM-DD') logger.info(f'downloading hourly Live Ocean forecast starting on {ymd}') base_url = config['temperature salinity']['download']['url'] dir_prefix = config['temperature salinity']['download']['directory prefix'] filename_tmpl = config['temperature salinity']['download']['file template'] url = f'{base_url}{dir_prefix}{yyyymmdd}/{filename_tmpl}' hours = config['temperature salinity']['download']['hours range'] dest_dir = Path(config['temperature salinity']['download']['dest dir'], yyyymmdd) grp_name = config['file group'] lib.mkdir(str(dest_dir), logger, grp_name=grp_name) checklist = {ymd: []} with requests.Session() as session: for hr in range(hours[0], hours[1] + 1): filepath = _get_file(url.format(hh=hr), filename_tmpl.format(hh=hr), dest_dir, session) salishsea_tools.UBC_subdomain.get_UBC_subdomain([str(filepath)]) subdomain_filepath = str(filepath).replace('.nc', '_UBC.nc') logger.debug(f'extracted UBC sub-domain: {subdomain_filepath}', extra={'subdomain_filepath': subdomain_filepath}) checklist[ymd].append(subdomain_filepath) filepath.unlink() nemo_cmd.api.deflate(dest_dir.glob('*.nc'), math.floor(multiprocessing.cpu_count() / 2)) return checklist
def _mkdirs(dest_dir_root, date, forecast, grp_name): lib.mkdir(os.path.join(dest_dir_root, date), logger, grp_name=grp_name) lib.mkdir( os.path.join(dest_dir_root, date, forecast), logger, grp_name=grp_name, exist_ok=False, )
def _move_file(expected_file, grib_forecast_dir, grp_name): """ :param :py:class:`pathlib.Path` expected_file: :param :py:class:`pathlib.Path` grib_forecast_dir: :param str grp_name: """ grib_hour_dir = grib_forecast_dir / expected_file.parent.stem lib.mkdir(grib_hour_dir, logger, grp_name=grp_name) shutil.move(os.fspath(expected_file), os.fspath(grib_hour_dir)) logger.debug(f"moved {expected_file} to {grib_hour_dir}/")
def collect_weather(parsed_args, config, *args): """ :param :py:class:`argparse.Namespace` parsed_args: :param :py:class:`nemo_nowcast.Config` config: :return: Nowcast system checklist items :rtype: dict """ forecast = parsed_args.forecast resolution = parsed_args.resolution.replace("km", " km") forecast_yyyymmdd = ( parsed_args.backfill_date.format("YYYYMMDD") if parsed_args.backfill else arrow.utcnow().shift(hours=-int(forecast) + 4).format("YYYYMMDD") ) datamart_dir = Path(config["weather"]["download"][resolution]["datamart dir"]) grib_dir = Path(config["weather"]["download"][resolution]["GRIB dir"]) grp_name = config["file group"] expected_files = _calc_expected_files( datamart_dir, forecast, forecast_yyyymmdd, resolution, config ) lib.mkdir(grib_dir / forecast_yyyymmdd, logger, grp_name=grp_name) logger.debug(f"created {grib_dir / forecast_yyyymmdd}/") lib.mkdir(grib_dir / forecast_yyyymmdd / forecast, logger, grp_name=grp_name) logger.debug(f"created {grib_dir / forecast_yyyymmdd/forecast}/") if parsed_args.backfill: logger.info( f"starting to move {parsed_args.backfill_date.format('YYYY-MM-DD')} files from {datamart_dir / forecast}/" ) for expected_file in expected_files: _move_file(expected_file, grib_dir / forecast_yyyymmdd / forecast, grp_name) else: handler = _GribFileEventHandler( expected_files, grib_dir / forecast_yyyymmdd / forecast, grp_name ) observer = watchdog.observers.Observer() observer.schedule(handler, os.fspath(datamart_dir / forecast), recursive=True) logger.info(f"starting to watch for files in {datamart_dir/forecast}/") observer.start() while expected_files: time.sleep(1) logger.info( f"finished collecting files from {datamart_dir/forecast}/ to " f"{grib_dir / forecast_yyyymmdd / forecast}/" ) checklist = { f"{forecast} {resolution.replace(' km', 'km')}": os.fspath( grib_dir / forecast_yyyymmdd / forecast ) } return checklist
def _copy_csv_to_results_dir(data_file, run_date, run_type, config): results_date = run_date if run_type == "nowcast" else run_date.shift( days=-1) results_dir = Path(config["results archive"][run_type], results_date.format("DDMMMYY").lower()) lib.mkdir(results_dir, logger, grp_name=config["file group"], exist_ok=True) shutil.copy2(data_file, results_dir) logger.debug(f"copied {data_file} to {results_dir}")
def get_grib(parsed_args, config, *args): forecast = parsed_args.forecast resolution = parsed_args.resolution.replace("km", " km") date = _calc_date(parsed_args, forecast) logger.info( f"downloading {forecast} {resolution} forecast GRIB2 files for {date}") dest_dir_root = config["weather"]["download"][resolution]["GRIB dir"] grp_name = config["file group"] _mkdirs(dest_dir_root, date, forecast, grp_name) url_tmpl = config["weather"]["download"][resolution]["url template"] filename_tmpl = config["weather"]["download"][resolution]["file template"] forecast_duration = config["weather"]["download"][resolution][ "forecast duration"] with requests.Session() as session: if parsed_args.no_verify_certs: session.verify = False for forecast_hour in range(1, forecast_duration + 1): hr_str = f"{forecast_hour:0=3}" lib.mkdir( os.path.join(dest_dir_root, date, forecast, hr_str), logger, grp_name=grp_name, exist_ok=False, ) for var in config["weather"]["download"][resolution][ "grib variables"]: filepath = _get_file( url_tmpl, filename_tmpl, var, dest_dir_root, date, forecast, hr_str, session, ) lib.fix_perms(filepath) checklist = { f"{forecast} {resolution.replace(' km', 'km')}": os.path.join(dest_dir_root, date, forecast) } return checklist
def get_grib(parsed_args, config, *args): forecast = parsed_args.forecast date = _calc_date(parsed_args, forecast) logger.info(f'downloading {forecast} forecast GRIB2 files for {date}', extra={'forecast': parsed_args.forecast}) dest_dir_root = config['weather']['GRIB dir'] grp_name = config['file group'] _mkdirs(dest_dir_root, date, forecast, grp_name) with requests.Session() as session: for forecast_hour in range(1, FORECAST_DURATION + 1): hr_str = f'{forecast_hour:0=3}' lib.mkdir(os.path.join(dest_dir_root, date, forecast, hr_str), logger, grp_name=grp_name, exist_ok=False) for var in GRIB_VARIABLES: filepath = _get_file(var, dest_dir_root, date, forecast, hr_str, session) os.chmod(filepath, FilePerms(user='******', group='rw', other='r')) checklist = {f'{date} {forecast} forecast': True} return checklist
def _render_figures(config, model, run_type, plot_type, dmy, fig_functions, test_figure_id): logger.info( f"starting to render {model} {run_type} {plot_type} {dmy} figures") checklist = {} fig_files = [] for svg_name, func in fig_functions.items(): fig_func = func["function"] args = func.get("args", []) kwargs = func.get("kwargs", {}) fig_save_format = func.get("format", "svg") image_loop_figure = func.get("image loop", False) test_figure = False if test_figure_id: test_figure = any(( svg_name == test_figure_id, image_loop_figure and svg_name.startswith(test_figure_id), fig_func.__module__.endswith(f"{plot_type}.{test_figure_id}"), )) if not test_figure: continue logger.debug(f"starting {fig_func.__module__}.{fig_func.__name__}") try: fig = _calc_figure(fig_func, args, kwargs) except (FileNotFoundError, IndexError, KeyError, TypeError): # **IMPORTANT**: the collection of exception above must match those # handled in the _calc_figure() function continue if test_figure: fig_files_dir = Path(config["figures"]["test path"], run_type, dmy) fig_files_dir.mkdir(parents=True, exist_ok=True) else: fig_files_dir = (Path(config["figures"]["storage path"], run_type, dmy) if model == "nemo" else Path( config["figures"]["storage path"], model, run_type, dmy)) lib.mkdir(fig_files_dir, logger, grp_name=config["file group"]) filename = fig_files_dir / f"{svg_name}_{dmy}.{fig_save_format}" if image_loop_figure: filename = fig_files_dir / f"{svg_name}.{fig_save_format}" fig.savefig(os.fspath(filename), facecolor=fig.get_facecolor(), bbox_inches="tight") logger.debug(f"{filename} saved") matplotlib.pyplot.close(fig) if fig_save_format == "svg": logger.debug(f"starting SVG scouring of {filename}") tmpfilename = filename.with_suffix(".scour") scour = Path(os.environ["NOWCAST_ENV"], "bin", "scour") cmd = f"{scour} {filename} {tmpfilename}" logger.debug(f"running subprocess: {cmd}") try: proc = subprocess.run( shlex.split(cmd), check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, ) except subprocess.CalledProcessError as e: logger.warning( "SVG scouring failed, proceeding with unscoured figure") logger.debug(f"scour return code: {e.returncode}") if e.output: logger.debug(e.output) continue logger.debug(proc.stdout) tmpfilename.rename(filename) logger.debug(f"{filename} scoured") lib.fix_perms(filename, grp_name=config["file group"]) fig_files.append(os.fspath(filename)) fig_path = _render_storm_surge_alerts_thumbnail( config, run_type, plot_type, dmy, fig, svg_name, fig_save_format, test_figure, ) if checklist is not None: checklist["storm surge alerts thumbnail"] = fig_path checklist[f"{model} {run_type} {plot_type}"] = fig_files logger.info( f"finished rendering {model} {run_type} {plot_type} {dmy} figures") return checklist
def make_surface_current_tiles(parsed_args, config, *args): """ :param :py:class:`argparse.Namespace` parsed_args: :param :py:class:`nemo_nowcast.Config` config: :return: Nowcast system checklist items :rtype: dict """ run_type = parsed_args.run_type run_date = parsed_args.run_date num_procs = parsed_args.nprocs dmy = run_date.format("DDMMMYY").lower() dmym1 = run_date.shift(days=-1).format("DDMMMYY").lower() ## TODO: Change to get results from ERDDAP rolling forecast for run_type == forecast* results_dir0 = ( Path( config["results archive"]["nowcast-green"], run_date.shift(days=+2).format("DDMMMYY").lower(), ) if run_type == "nowcast-green" else Path(config["results archive"][run_type], dmy) ) if run_type == "nowcast-green": results_dirm1 = Path( config["results archive"]["nowcast-green"], run_date.shift(days=+1).format("DDMMMYY").lower(), ) results_dirm2 = Path( config["results archive"]["nowcast-green"], run_date.format("DDMMMYY").lower(), ) if run_type == "forecast": results_dirm1 = Path(config["results archive"]["nowcast"], dmy) results_dirm2 = Path(config["results archive"]["nowcast"], dmym1) if run_type == "forecast2": results_dirm1 = Path(config["results archive"]["forecast"], dmy) results_dirm2 = Path(config["results archive"]["nowcast"], dmy) grid_dir = Path(config["figures"]["grid dir"]) coordf = grid_dir / config["run types"][run_type]["coordinates"] mesh_maskf = grid_dir / config["run types"][run_type]["mesh mask"] bathyf = grid_dir / config["run types"][run_type]["bathymetry"] storage_path = Path( config["figures"]["surface current tiles"]["storage path"], run_type, dmy ) lib.mkdir(storage_path, logger, grp_name=config["file group"]) # Loop over last 48h and this forecast{,2} logger.info( f"starting rendering of tiles for {run_date.format('YYYY-MM-DD')} {run_type} " f"in {num_procs} processes into {storage_path}" ) for results_dir in [results_dirm2, results_dirm1, results_dir0]: u_list = glob(os.fspath(results_dir) + "/SalishSea_1h_*_grid_U.nc") v_list = glob(os.fspath(results_dir) + "/SalishSea_1h_*_grid_V.nc") Uf = Path(u_list[0]) Vf = Path(v_list[0]) with netCDF4.Dataset(Uf) as dsU: max_time_index = dsU.dimensions["time_counter"].size units = dsU.variables["time_counter"].units calendar = dsU.variables["time_counter"].calendar sec = dsU.variables["time_counter"][:] expansion_factor = 0.1 # 10% overlap for each tile logger.debug(f"creating figures using {num_procs} concurrent process(es)") # Add tasks to a joinable queue q = multiprocessing.JoinableQueue() for t_index in range(max_time_index): task = ( t_index, sec, units, calendar, run_date, Uf, Vf, coordf, mesh_maskf, bathyf, tile_coords_dic, expansion_factor, storage_path, ) q.put(task) # Spawn a set of worker processes procs = [] for i in range(num_procs): p = multiprocessing.Process(target=_process_time_slice, args=(q,)) procs.append(p) # Start each one for p in procs: p.start() # Wait until they complete for p in procs: p.join() # Close the queue q.close() _pdf_concatenate(storage_path, tile_coords_dic) checklist = { run_type: { "run date": run_date.format("YYYY-MM-DD"), "png": sorted( [os.fspath(f) for f in storage_path.iterdir() if f.suffix == ".png"] ), "pdf": sorted( [os.fspath(f) for f in storage_path.iterdir() if f.suffix == ".pdf"] ), } } logger.info( f"finished rendering of tiles for {run_date.format('YYYY-MM-DD')} {run_type} " f"into {storage_path}" ) return checklist
def get_NeahBay_ssh(parsed_args, config, *args): """Generate sea surface height forcing files from the Neah Bay storm surge website. """ run_type = parsed_args.run_type with nc.Dataset(config['coordinates']) as coordinates: lats = coordinates.variables['nav_lat'][:] lons = coordinates.variables['nav_lon'][:] logger.debug(f'loaded lats & lons from {config["coordinates"]}', extra={'run_type': run_type}) # Scrape the surge data from the website into a text file, # store the file in the run results directory, # and load the data for processing into netCDF4 files utc_now = datetime.datetime.now(pytz.timezone('UTC')) textfile = _read_website(config['ssh']['ssh dir']) lib.fix_perms(textfile, grp_name=config['file group']) checklist = {'txt': os.path.basename(textfile)} # Store a copy of the text file in the run results directory so that # there is definitive record of the sea surface height data that was # used for the run run_date = _utc_now_to_run_date(utc_now, run_type) results_dir = os.path.join(config['results archive'][run_type], run_date.strftime('%d%b%y').lower()) lib.mkdir(results_dir, logger, grp_name=config['file group'], exist_ok=True) shutil.copy2(textfile, results_dir) # Grab all surge data in the textfile dates, sshs, fflags = residuals.NeahBay_forcing_anom( textfile, run_date, config['ssh']['tidal predictions']) # Identify days with full ssh information dates_full = _list_full_days(dates, sshs, fflags) # Set up plotting fig, ax = _setup_plotting() # Loop through full days and save netcdf ip = 0 for d in dates_full: tc, _, sshd, fflagd = _isolate_day(d, dates, sshs, fflags) forecast_flag = fflagd.any() # Plotting if ip < 3: ax.plot(sshd, '-o', lw=2, label=d.strftime('%d-%b-%Y')) ip = ip + 1 filepath = _save_netcdf(d, tc, sshd, forecast_flag, textfile, config, lats, lons) filename = os.path.basename(filepath) lib.fix_perms(filename, grp_name=config['file group']) if forecast_flag: if 'fcst' in checklist: checklist['fcst'].append(filename) else: checklist['fcst'] = [filename] else: checklist['obs'] = filename ax.legend(loc=4) image_file = config['ssh']['monitoring image'] canvas = matplotlib.backends.backend_agg.FigureCanvasAgg(fig) canvas.print_figure(image_file) lib.fix_perms(image_file, grp_name=config['file group']) return checklist
def _render_figures(config, run_type, plot_type, dmy, fig_functions, test_figure_id): checklist = {} fig_files = [] for svg_name, func in fig_functions.items(): fig_func = func['function'] args = func.get('args', []) kwargs = func.get('kwargs', {}) fig_save_format = func.get('format', 'svg') test_figure = False if test_figure_id: test_figure = any(( svg_name == test_figure_id, fig_func.__module__.endswith(f'{plot_type}.{test_figure_id}'), # legacy: for figures.figures module functions fig_func.__name__ == test_figure_id, )) if not test_figure: continue logger.debug(f'starting {fig_func.__module__}.{fig_func.__name__}') try: fig = _calc_figure(fig_func, args, kwargs) except (FileNotFoundError, IndexError, KeyError, TypeError): # **IMPORTANT**: the collection of exception above must match those # handled in the _calc_figure() function continue if test_figure: fig_files_dir = Path(config['figures']['test path'], run_type, dmy) fig_files_dir.mkdir(parents=True, exist_ok=True) else: fig_files_dir = Path(config['figures']['storage path'], run_type, dmy) lib.mkdir(os.fspath(fig_files_dir), logger, grp_name=config['file group']) filename = fig_files_dir / f'{svg_name}_{dmy}.{fig_save_format}' fig.savefig(os.fspath(filename), facecolor=fig.get_facecolor(), bbox_inches='tight') lib.fix_perms(os.fspath(filename), grp_name=config['file group']) logger.info(f'{filename} saved') try: cmd = "scour " + str(filename) + " " + str(filename) + ".scour.svg" r = subprocess.run(shlex.split(cmd)) if r.returncode == 0: os.rename(str(filename) + ".scour.svg", str(filename)) lib.fix_perms(os.fspath(filename), grp_name=config['file group']) logger.info(f'scoured {filename} saved') else: logger.warning( f'Scouring failed, proceeded with unscoured image') except (FileNotFoundError, TypeError): logger.warning('Scouring failed,proceeded with unscoured image.') fig_files.append(os.fspath(filename)) fig_path = _render_storm_surge_alerts_thumbnail( config, run_type, plot_type, dmy, fig, svg_name, fig_save_format, test_figure) if checklist is not None: checklist['storm surge alerts thumbnail'] = fig_path checklist[f'{run_type} {plot_type}'] = fig_files return checklist