with NoStdStreams(): tmpd = [parse_date(str(d)) for d in vardict['datetime']] vardict['datetime'] = tmpd del tmpd if transform_lons==180: # transform longitudes from -180 to 180 vardict['longitude'] = ((vardict['longitude'] - 180) % 360) - 180 elif transform_lons==360: print('not yet implemented !!') return vardict, fc_date, leadtime, filestr, filevarname # ---------------------------------------------------------------------# # read yaml config files: model_dict = load_or_default('model_specs.yaml') variable_info = load_or_default('variable_info.yaml') class model_class(): ''' class to read and process model data model: e.g. Hs[time,lat,lon], lat[rlat,rlon], lon[rlat,rlon] This class should communicate with the satellite, model, and station classes. ''' def __init__(self, model='mwam4', sdate=None, edate=None, date_incr=1,
import sys import os from datetime import datetime import pytest from wavy.wconfig import load_or_default import wavy.sat_collectors from wavy.satmod import satellite_class as sc satellite_dict = load_or_default('satellite_specs.yaml') #def test_ftp_files_and_satellite_class_features(tmpdir): @pytest.mark.need_credentials def test_collectors_cmems_L3(tmpdir): sdate = "2020-1-2 12" edate = "2020-1-2 12" sdate_dt = datetime(2020, 1, 2, 12) edate_dt = datetime(2020, 1, 2, 12) twin = 30 nproc = 1 mission = 's3a' product = 'cmems_L3_NRT' # evoke fct get_remote_files api_url = None dict_for_sub = {'mission': mission} wavy.sat_collectors.get_remote_files(path_local=tmpdir, sdate=sdate_dt, edate=edate_dt, twin=twin, nproc=nproc,
""" Module for quicklook fct """ # imports import numpy as np import os from mpl_toolkits.axes_grid1.inset_locator import inset_axes # own imports from wavy.wconfig import load_or_default # read yaml config files: region_dict = load_or_default('region_specs.yaml') variable_info = load_or_default('variable_info.yaml') model_dict = load_or_default('model_specs.yaml') quicklook_dict = load_or_default('quicklook_specs.yaml') # define global functions def comp_fig(sa_obj=None, mc_obj=None, coll_obj=None, **kwargs): import matplotlib.cm as mplcm import matplotlib as mpl import matplotlib.pyplot as plt import numpy as np import cartopy.crs as ccrs import cartopy.feature as cfeature import cmocean from cartopy.mpl.gridliner import LONGITUDE_FORMATTER, LATITUDE_FORMATTER import matplotlib.ticker as mticker from cartopy.mpl.ticker import LongitudeFormatter, LatitudeFormatter
def main(): # read yaml config files: satellite_dict = load_or_default('satellite_specs.yaml') # parser parser = argparse.ArgumentParser(description=""" Download satellite nc-files Usage: ./wavyDownload.py -sat s3a -sd 2020100100 -ed 2020101000 """, formatter_class=RawTextHelpFormatter) parser.add_argument("-sd", metavar='startdate', help="start date of time period to be downloaded") parser.add_argument("-ed", metavar='enddate', help="end date of time period to be downloaded") parser.add_argument("-sat", metavar='satellite', help="satellite mission, currently available\n \ \ncmems_L3_NRT:\ \n s3a - Sentinel-3A\ \n s3b - Sentinel-3B\ \n j3 - Jason-3 (reference mission)\ \n c2 - Cryosat-2\ \n al - SARAL/AltiKa\ \n cfo - CFOSAT\ \n h2b - HaiYang-2B\ \n\ \ncmems_L3_s6a:\ \n s6a - Sentinel-6A Michael Freilich\ \n\ \neumetsat_L2:\ \n s3a - Sentinel-3A\ \n s3b - Sentinel-3B\ \n\ \ncci_L2P:\ \n j1 - Jason-1\ \n j2 - Jason-2\ \n j3 - Jason-3\ \n c2 - Cryosat-2\ \n envisat - Envisat\ \n ers1 - European Remote-Sensing Satellite-1\ \n ers2 - European Remote-Sensing Satellite-2\ \n topex - TOPEX/Poseidon\ \n al - SARAL/AltiKa\ \n gfo - GEOSAT Follow-On\ \n\ \ncci_L3:\ \n multi - multimission product 1991-2018 \n\ \n") parser.add_argument("-path", metavar='path', help="destination for downloaded data") parser.add_argument("-product", metavar='product', help="product name as specified in *_specs.yaml") parser.add_argument("-api_url", metavar='api_url', help="source of eumetsat L2 data") parser.add_argument("-nproc", metavar='nproc', help="number of possible simultaneous downloads", type=int) args = parser.parse_args() # settings now = datetime.now() if args.sat is None: satlst = ['s3a'] elif args.sat == 'cmems_L3_NRT': satlst = satellite_dict['cmems_L3_NRT']['mission'].keys() else: satlst = [args.sat] if args.sd is None: sdate = datetime(now.year, now.month, now.day, now.hour) - timedelta(hours=24) else: sdate = datetime(int(args.sd[0:4]), int(args.sd[4:6]), int(args.sd[6:8]), int(args.sd[8:10])) if args.ed is None: edate = datetime(now.year, now.month, now.day, now.hour, now.minute) else: edate = datetime(int(args.ed[0:4]), int(args.ed[4:6]), int(args.ed[6:8]), int(args.ed[8:10])) if args.nproc is None: args.nproc = 1 if args.product is None: args.product = 'cmems_L3_NRT' print(args) twin = 30 for sat in satlst: for i in range(1): # try: print("Attempting to download data for:", sat) print("Time period:", str(sdate), "to", str(edate)) start_time = time.time() dict_for_sub = {'mission': sat} #path_local,sdate,edate,twin,nproc,product,api_url,sat,dict_for_sub #sa_obj = get_remote_files(\ # args.path, sdate, edate, twin, # args.nproc, args.product, # args.api_url, sat, # dict_for_sub) sa_obj = get_remote_files(\ path_local=args.path, sdate=sdate,edate=edate, twin=twin,nproc=args.nproc, product=args.product, api_url=args.api_url, mission=sat, dict_for_sub=dict_for_sub) time1 = time.time() - start_time print("Time used for collecting data: ", time1, " seconds")
from wavy.utils import make_subdict from wavy.utils import parse_date from wavy.utils import haversineA from wavy.utils import flatten from wavy.wconfig import load_or_default from wavy.modelmod import make_model_filename_wrapper from wavy.modelmod import get_model_filedate from wavy.modelmod import model_class, get_model from wavy.ncmod import dumptonc_ts_collocation from wavy.ncmod import ncdumpMeta, get_filevarname from wavy.satmod import satellite_class from wavy.insitumod import insitu_class # ---------------------------------------------------------------------# # read yaml config files: model_dict = load_or_default('model_specs.yaml') insitu_dict = load_or_default('insitu_specs.yaml') collocation_dict = load_or_default('collocation_specs.yaml') variable_info = load_or_default('variable_info.yaml') def collocation_fct(obs_lons, obs_lats, model_lons, model_lats): grid = pyresample.geometry.GridDefinition(\ lats=model_lats, \ lons=model_lons) # Define some sample points swath = pyresample.geometry.SwathDefinition(lons=obs_lons, lats=obs_lats) # Determine nearest (great circle distance) neighbour in the grid. valid_input_index, valid_output_index, index_array, distance_array = \ pyresample.kd_tree.get_neighbour_info( source_geo_def=grid,
from wavy.utils import find_included_times, NoStdStreams from wavy.utils import parse_date from wavy.utils import make_pathtofile, make_subdict from wavy.utils import finditem, haversineA from wavy.utils import flatten from wavy.modelmod import make_model_filename_wrapper from wavy.modelmod import read_model_nc_output_lru from wavy.wconfig import load_or_default from wavy.filtermod import filter_main, vardict_unique from wavy.filtermod import rm_nan_from_vardict from wavy.sat_collectors import get_remote_files from wavy.sat_readers import read_local_files # ---------------------------------------------------------------------# # read yaml config files: region_dict = load_or_default('region_specs.yaml') model_dict = load_or_default('model_specs.yaml') satellite_dict = load_or_default('satellite_specs.yaml') variable_info = load_or_default('variable_info.yaml') # --- global functions ------------------------------------------------# def get_local_files(sdate, edate, twin, product, dict_for_sub=None, path_local=None): """ Function to retrieve list of files/paths for available
import numpy as np from copy import deepcopy import netCDF4 import pandas as pd import roaring_landmask from scipy.stats import circmean from datetime import timedelta from wavy.utils import flatten # own imports from wavy.utils import find_included_times, collocate_times from wavy.wconfig import load_or_default ROAR = None variable_info = load_or_default('variable_info.yaml') def filter_main(vardict_in,varalias='Hs',**kwargs): """ Governing function of filtermod Tasks: - check if prior/post transforms are needed - check if cleaning is needed - check if filter is needed - check if land mask is needed - if so apply cleaning/filters to subsets i.e. each chunk will be fed into filter_data and consolidated when finished with all chunks Args: vardict
import numpy as np from datetime import datetime import os import sys from copy import deepcopy from functools import lru_cache from tqdm import tqdm import zipfile import tempfile # own imports from wavy.wconfig import load_or_default from wavy.utils import find_included_times, finditem # read yaml config files: insitu_dict = load_or_default('insitu_specs.yaml') variable_info = load_or_default('variable_info.yaml') # --- global functions ------------------------------------------------# """ definition of some global functions """ # currently None # ---------------------------------------------------------------------# def check_if_ncfile_accessible(fstr): # remove escape character because xarray handles white spaces # but cannot handle escape characters (apparently) fstr_repl=fstr.replace('\\','') try: xs=xr.open_dataset(fstr_repl)
def main(): variable_info = load_or_default('variable_info.yaml') model_dict = load_or_default('model_specs.yaml') # parser parser = argparse.ArgumentParser(description=""" Check availability of satellite SWH data. Example: ./wavyQuick.py -sat s3a -reg mwam4 -mod mwam4 -sd 2020100112 -lt 0 -twin 30 --col --show ./wavyQuick.py -sat s3a -reg mwam4 -sd 2020100100 -ed 2020101000 --show ./wavyQuick.py -sat s3a -reg mwam4 -sd 2020100100 -ed 2020101000 -dump ./test.nc """, formatter_class=RawTextHelpFormatter) parser.add_argument("-reg", metavar='region', help="region to check") parser.add_argument("-sat", metavar='satellite', help="satellite mission, currently available\n \ \ncmems_L3_NRT:\ \n s3a - Sentinel-3A\ \n s3b - Sentinel-3B\ \n j3 - Jason-3 (reference mission)\ \n c2 - Cryosat-2\ \n al - SARAL/AltiKa\ \n cfo - CFOSAT\ \n h2b - HaiYang-2B\ \n\ \ncmems_L3_s6a:\ \n s6a - Sentinel-6A Michael Freilich\ \n\ \neumetsat_L2:\ \n s3a - Sentinel-3A\ \n s3b - Sentinel-3B\ \n\ \ncci_L2P:\ \n j1 - Jason-1\ \n j2 - Jason-2\ \n j3 - Jason-3\ \n c2 - Cryosat-2\ \n envisat - Envisat\ \n ers1 - European Remote-Sensing Satellite-1\ \n ers2 - European Remote-Sensing Satellite-2\ \n topex - TOPEX/Poseidon\ \n al - SARAL/AltiKa\ \n gfo - GEOSAT Follow-On\ \n\ \ncci_L3:\ \n multi - multimission product 1991-2018\ \n\ \nother options are:\ \n all - for all cmems satellites\ \n list - a list of chosen satellites using -l\n \ \n") parser.add_argument("-product", metavar='product', help="available products as specified in *_specs.yaml") parser.add_argument('-l', metavar='satellite list', help='delimited list input for sats', type=str) parser.add_argument("-sd", metavar='startdate', help="start date of time period to check") parser.add_argument("-ed", metavar='enddate', help="end date of time period to check") parser.add_argument("-mod", metavar='model', help="chosen wave model") parser.add_argument("-var", metavar='varalias', help="alias for chosen variable") parser.add_argument("-lt", metavar='lead time', type=int, help="lead time from initialization") parser.add_argument("-twin", metavar='time window', type=int, help="time window for collocation") parser.add_argument("-dist", metavar='distance limit', type=int, help="distance limit for collocation") parser.add_argument("--col", metavar="collocation", help="collocation", action='store_const', const=True) parser.add_argument("--show", help="show figure", action='store_const', const=True) parser.add_argument("-savep", metavar="savepath", help="save figure to path") parser.add_argument("-dump", metavar="outpath", help="dump data to .nc-file") args = parser.parse_args() print("Parsed arguments: ", args) flatten = lambda l: [item for sublist in l for item in sublist] # setup if args.var is None: args.var = 'Hs' sdate = datetime(int(args.sd[0:4]), int(args.sd[4:6]), int(args.sd[6:8]), int(args.sd[8:10])) if args.product is None: args.product = 'cmems_L3_NRT' if args.twin is None: args.twin = 30 if args.dist is None: args.dist = 10 if args.ed is None: edate = sdate else: edate = datetime(int(args.ed[0:4]), int(args.ed[4:6]), int(args.ed[6:8]), int(args.ed[8:10])) twin = 0 if (args.reg is None and args.mod is not None): args.reg = args.mod # get data if args.sat == 'all': satlist = ['s3a', 's3b', 'j3', 'c2', 'al', 'cfo', 'h2b'] lats = [] lons = [] var = [] time = [] dtime = [] sats = [] satnamelst = [] for sat in satlist: try: sa_obj_tmp = sa(sdate=sdate, mission=sat, edate=edate, twin=args.twin, region=args.reg, varalias=args.var, product=args.product) if ('vars' in vars(sa_obj_tmp).keys() and len(sa_obj_tmp.vars['time']) > 0): sa_obj = sa_obj_tmp lats.append(sa_obj.vars['latitude']) lons.append(sa_obj.vars['longitude']) var.append(sa_obj.vars[variable_info[args.var]\ ['standard_name']]) time.append(sa_obj.vars['time']) dtime.append(sa_obj.vars['datetime']) sats.append(sat) satnamelst.append([sat] * len(sa_obj.vars['time'])) except Exception as e: print(e) print(sat + ' not available') pass lats = flatten(lats) lons = flatten(lons) var = flatten(var) time = flatten(time) dtime = flatten(dtime) satnames = flatten(satnamelst) sa_obj.vars['latitude'] = np.array(lats) sa_obj.vars['longitude'] = np.array(lons) sa_obj.vars[variable_info[args.var]['standard_name']] = np.array(var) sa_obj.vars['time'] = time sa_obj.vars['datetime'] = dtime sa_obj.region = args.reg sa_obj.mission = str(sats) sa_obj.mission_ts = satnames elif args.sat == 'list': satlist = args.l.split(',') lats = [] lons = [] var = [] time = [] sats = [] satnamelst = [] for sat in satlist: try: sa_obj_tmp = sa(sdate=sdate, mission=sat, edate=edate, twin=args.twin, region=args.reg, varalias=args.var, product=args.product) if ('vars' in vars(sa_obj_tmp).keys() and len(sa_obj_tmp.vars['time']) > 0): sa_obj = sa_obj_tmp lats.append(sa_obj.vars['latitude']) lons.append(sa_obj.vars['longitude']) var.append(sa_obj.vars[variable_info[args.var]\ ['standard_name']]) time.append(sa_obj.vars['time']) sats.append(sat) satnamelst.append([sat] * len(sa_obj.vars['time'])) except Exception as e: print(e) print(sat + ' not available') pass lats = flatten(lats) lons = flatten(lons) var = flatten(var) time = flatten(time) satnames = flatten(satnamelst) sa_obj.vars['time'] = time sa_obj.vars['latitude'] = np.array(lats) sa_obj.vars['longitude'] = np.array(lons) sa_obj.vars[variable_info[args.var]['standard_name']] = np.array(var) sa_obj.region = args.reg sa_obj.mission = str(sats) sa_obj.mission_ts = satnames else: sa_obj = sa(sdate=sdate, mission=args.sat, edate=edate, twin=args.twin, region=args.reg, varalias=args.var, product=args.product) # plot if (args.mod is None and sa_obj.region not in model_dict): plot_sat(sa_obj, savepath=args.savep, showfig=args.show) elif (args.mod is None and sa_obj.region in model_dict): print('Chosen region is a specified model domain') mc_obj = mc(model=sa_obj.region, fc_date=model_dict[sa_obj.region]['grid_date'], varalias=args.var) plot_sat(sa_obj, mc_obj=mc_obj, savepath=args.savep, showfig=args.show) elif (args.mod is not None and args.col is True and sdate == edate): # get model collocated values mc_obj = mc(model=args.mod, fc_date=edate, leadtime=args.lt, varalias=args.var) #collocation coll_obj = coll(mc_obj_in=mc_obj, obs_obj_in=sa_obj, distlim=args.dist, leadtime=args.lt) valid_dict = validate(coll_obj.vars) disp_validation(valid_dict) comp_fig(sa_obj=sa_obj, mc_obj=mc_obj, coll_obj=coll_obj, savepath=args.savep, showfig=args.show) elif (args.mod is not None and args.col is False): # get model collocated values mc_obj = mc(model=sa_obj.region, fc_date=edate, leadtime=args.lt, varalias=args.var) results_dict = {'valid_date':[edate], 'date_matches':[edate-timedelta(minutes=args.twin), edate+timedelta(minutes=args.twin)], 'model_lons_matches':sa_obj.vars['longitude'], 'model_lats_matches':sa_obj.vars['latitude'], 'sat_matches':sa_obj.vars[variable_info[args.var]\ ['standard_name']]} comp_fig(args.mod, sa_obj, mc_obj.vars[variable_info[args.var]['standard_name']], mc_obj.vars['longitude'], mc_obj.vars['latitude'], results_dict, variable_info[args.var]['standard_name'], savepath=args.savep, showfig=args.show) elif (args.mod is not None and args.col is True and args.sat is not None): #collocation date_incr = model_dict[args.mod].get('date_incr', 1) coll_obj = coll(model=args.mod, obs_obj_in=sa_obj, distlim=args.dist, leadtime=args.lt, date_incr=date_incr) valid_dict = coll_obj.validate_collocated_values() if args.show is True: print( "Figure not shown! Request is ambiguous for multiple model time steps." ) # dump to .ncfile if args.dump is not None: if args.col is not None: coll_obj.write_to_nc(pathtofile=args.dump) else: sa_obj.write_to_nc(pathtofile=args.dump)
import zipfile import tempfile from tqdm import tqdm import xarray as xr from datetime import timedelta import netCDF4 # own imports from wavy.ncmod import ncdumpMeta, get_filevarname from wavy.ncmod import read_netcdfs, read_netcdfs_zipped_lru from wavy.wconfig import load_or_default from wavy.utils import parse_date # ---------------------------------------------------------------------# # read yaml config files: satellite_dict = load_or_default('satellite_specs.yaml') variable_info = load_or_default('variable_info.yaml') def unzip_eumetsat(pathlst, tmpdir): """ Function to unzip eumetsat files prior to reading param: pathlst - list of paths to zipped files tmpdir - temporary folder to unzipped files return: pathlst_new - new list of paths to unzipped files """ for count, f in enumerate(pathlst):