with NoStdStreams():
tmpd = [parse_date(str(d)) for d in vardict['datetime']]
vardict['datetime'] = tmpd
del tmpd
if transform_lons==180:
# transform longitudes from -180 to 180
vardict['longitude'] = ((vardict['longitude'] - 180) % 360) - 180
elif transform_lons==360:
print('not yet implemented !!')
return vardict, fc_date, leadtime, filestr, filevarname
# ---------------------------------------------------------------------#
# read yaml config files:
model_dict = load_or_default('model_specs.yaml')
variable_info = load_or_default('variable_info.yaml')
class model_class():
'''
class to read and process model data
model: e.g. Hs[time,lat,lon], lat[rlat,rlon], lon[rlat,rlon]
This class should communicate with the satellite, model, and
station classes.
'''
def __init__(self,
model='mwam4',
sdate=None,
edate=None,
date_incr=1,
import sys
import os
from datetime import datetime
import pytest
from wavy.wconfig import load_or_default
import wavy.sat_collectors
from wavy.satmod import satellite_class as sc
satellite_dict = load_or_default('satellite_specs.yaml')
#def test_ftp_files_and_satellite_class_features(tmpdir):
@pytest.mark.need_credentials
def test_collectors_cmems_L3(tmpdir):
sdate = "2020-1-2 12"
edate = "2020-1-2 12"
sdate_dt = datetime(2020, 1, 2, 12)
edate_dt = datetime(2020, 1, 2, 12)
twin = 30
nproc = 1
mission = 's3a'
product = 'cmems_L3_NRT'
# evoke fct get_remote_files
api_url = None
dict_for_sub = {'mission': mission}
wavy.sat_collectors.get_remote_files(path_local=tmpdir,
sdate=sdate_dt,
edate=edate_dt,
twin=twin,
nproc=nproc,
"""
Module for quicklook fct
"""
# imports
import numpy as np
import os
from mpl_toolkits.axes_grid1.inset_locator import inset_axes
# own imports
from wavy.wconfig import load_or_default
# read yaml config files:
region_dict = load_or_default('region_specs.yaml')
variable_info = load_or_default('variable_info.yaml')
model_dict = load_or_default('model_specs.yaml')
quicklook_dict = load_or_default('quicklook_specs.yaml')
# define global functions
def comp_fig(sa_obj=None, mc_obj=None, coll_obj=None, **kwargs):
import matplotlib.cm as mplcm
import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import cartopy.crs as ccrs
import cartopy.feature as cfeature
import cmocean
from cartopy.mpl.gridliner import LONGITUDE_FORMATTER, LATITUDE_FORMATTER
import matplotlib.ticker as mticker
from cartopy.mpl.ticker import LongitudeFormatter, LatitudeFormatter
def main():
# read yaml config files:
satellite_dict = load_or_default('satellite_specs.yaml')
# parser
parser = argparse.ArgumentParser(description="""
Download satellite nc-files
Usage:
./wavyDownload.py -sat s3a -sd 2020100100 -ed 2020101000
""",
formatter_class=RawTextHelpFormatter)
parser.add_argument("-sd",
metavar='startdate',
help="start date of time period to be downloaded")
parser.add_argument("-ed",
metavar='enddate',
help="end date of time period to be downloaded")
parser.add_argument("-sat",
metavar='satellite',
help="satellite mission, currently available\n \
\ncmems_L3_NRT:\
\n s3a - Sentinel-3A\
\n s3b - Sentinel-3B\
\n j3 - Jason-3 (reference mission)\
\n c2 - Cryosat-2\
\n al - SARAL/AltiKa\
\n cfo - CFOSAT\
\n h2b - HaiYang-2B\
\n\
\ncmems_L3_s6a:\
\n s6a - Sentinel-6A Michael Freilich\
\n\
\neumetsat_L2:\
\n s3a - Sentinel-3A\
\n s3b - Sentinel-3B\
\n\
\ncci_L2P:\
\n j1 - Jason-1\
\n j2 - Jason-2\
\n j3 - Jason-3\
\n c2 - Cryosat-2\
\n envisat - Envisat\
\n ers1 - European Remote-Sensing Satellite-1\
\n ers2 - European Remote-Sensing Satellite-2\
\n topex - TOPEX/Poseidon\
\n al - SARAL/AltiKa\
\n gfo - GEOSAT Follow-On\
\n\
\ncci_L3:\
\n multi - multimission product 1991-2018 \n\
\n")
parser.add_argument("-path",
metavar='path',
help="destination for downloaded data")
parser.add_argument("-product",
metavar='product',
help="product name as specified in *_specs.yaml")
parser.add_argument("-api_url",
metavar='api_url',
help="source of eumetsat L2 data")
parser.add_argument("-nproc",
metavar='nproc',
help="number of possible simultaneous downloads",
type=int)
args = parser.parse_args()
# settings
now = datetime.now()
if args.sat is None:
satlst = ['s3a']
elif args.sat == 'cmems_L3_NRT':
satlst = satellite_dict['cmems_L3_NRT']['mission'].keys()
else:
satlst = [args.sat]
if args.sd is None:
sdate = datetime(now.year, now.month, now.day,
now.hour) - timedelta(hours=24)
else:
sdate = datetime(int(args.sd[0:4]), int(args.sd[4:6]),
int(args.sd[6:8]), int(args.sd[8:10]))
if args.ed is None:
edate = datetime(now.year, now.month, now.day, now.hour, now.minute)
else:
edate = datetime(int(args.ed[0:4]), int(args.ed[4:6]),
int(args.ed[6:8]), int(args.ed[8:10]))
if args.nproc is None:
args.nproc = 1
if args.product is None:
args.product = 'cmems_L3_NRT'
print(args)
twin = 30
for sat in satlst:
for i in range(1):
# try:
print("Attempting to download data for:", sat)
print("Time period:", str(sdate), "to", str(edate))
start_time = time.time()
dict_for_sub = {'mission': sat}
#path_local,sdate,edate,twin,nproc,product,api_url,sat,dict_for_sub
#sa_obj = get_remote_files(\
# args.path, sdate, edate, twin,
# args.nproc, args.product,
# args.api_url, sat,
# dict_for_sub)
sa_obj = get_remote_files(\
path_local=args.path,
sdate=sdate,edate=edate,
twin=twin,nproc=args.nproc,
product=args.product,
api_url=args.api_url,
mission=sat,
dict_for_sub=dict_for_sub)
time1 = time.time() - start_time
print("Time used for collecting data: ", time1, " seconds")
from wavy.utils import make_subdict
from wavy.utils import parse_date
from wavy.utils import haversineA
from wavy.utils import flatten
from wavy.wconfig import load_or_default
from wavy.modelmod import make_model_filename_wrapper
from wavy.modelmod import get_model_filedate
from wavy.modelmod import model_class, get_model
from wavy.ncmod import dumptonc_ts_collocation
from wavy.ncmod import ncdumpMeta, get_filevarname
from wavy.satmod import satellite_class
from wavy.insitumod import insitu_class
# ---------------------------------------------------------------------#
# read yaml config files:
model_dict = load_or_default('model_specs.yaml')
insitu_dict = load_or_default('insitu_specs.yaml')
collocation_dict = load_or_default('collocation_specs.yaml')
variable_info = load_or_default('variable_info.yaml')
def collocation_fct(obs_lons, obs_lats, model_lons, model_lats):
grid = pyresample.geometry.GridDefinition(\
lats=model_lats, \
lons=model_lons)
# Define some sample points
swath = pyresample.geometry.SwathDefinition(lons=obs_lons, lats=obs_lats)
# Determine nearest (great circle distance) neighbour in the grid.
valid_input_index, valid_output_index, index_array, distance_array = \
pyresample.kd_tree.get_neighbour_info(
source_geo_def=grid,
from wavy.utils import find_included_times, NoStdStreams
from wavy.utils import parse_date
from wavy.utils import make_pathtofile, make_subdict
from wavy.utils import finditem, haversineA
from wavy.utils import flatten
from wavy.modelmod import make_model_filename_wrapper
from wavy.modelmod import read_model_nc_output_lru
from wavy.wconfig import load_or_default
from wavy.filtermod import filter_main, vardict_unique
from wavy.filtermod import rm_nan_from_vardict
from wavy.sat_collectors import get_remote_files
from wavy.sat_readers import read_local_files
# ---------------------------------------------------------------------#
# read yaml config files:
region_dict = load_or_default('region_specs.yaml')
model_dict = load_or_default('model_specs.yaml')
satellite_dict = load_or_default('satellite_specs.yaml')
variable_info = load_or_default('variable_info.yaml')
# --- global functions ------------------------------------------------#
def get_local_files(sdate,
edate,
twin,
product,
dict_for_sub=None,
path_local=None):
"""
Function to retrieve list of files/paths for available
import numpy as np
from copy import deepcopy
import netCDF4
import pandas as pd
import roaring_landmask
from scipy.stats import circmean
from datetime import timedelta
from wavy.utils import flatten
# own imports
from wavy.utils import find_included_times, collocate_times
from wavy.wconfig import load_or_default
ROAR = None
variable_info = load_or_default('variable_info.yaml')
def filter_main(vardict_in,varalias='Hs',**kwargs):
""" Governing function of filtermod
Tasks:
- check if prior/post transforms are needed
- check if cleaning is needed
- check if filter is needed
- check if land mask is needed
- if so apply cleaning/filters to subsets
i.e. each chunk will be fed into filter_data
and consolidated when finished with all chunks
Args:
vardict
import numpy as np
from datetime import datetime
import os
import sys
from copy import deepcopy
from functools import lru_cache
from tqdm import tqdm
import zipfile
import tempfile
# own imports
from wavy.wconfig import load_or_default
from wavy.utils import find_included_times, finditem
# read yaml config files:
insitu_dict = load_or_default('insitu_specs.yaml')
variable_info = load_or_default('variable_info.yaml')
# --- global functions ------------------------------------------------#
"""
definition of some global functions
"""
# currently None
# ---------------------------------------------------------------------#
def check_if_ncfile_accessible(fstr):
# remove escape character because xarray handles white spaces
# but cannot handle escape characters (apparently)
fstr_repl=fstr.replace('\\','')
try:
xs=xr.open_dataset(fstr_repl)
def main():
variable_info = load_or_default('variable_info.yaml')
model_dict = load_or_default('model_specs.yaml')
# parser
parser = argparse.ArgumentParser(description="""
Check availability of satellite SWH data. Example:
./wavyQuick.py -sat s3a -reg mwam4 -mod mwam4 -sd 2020100112 -lt 0 -twin 30 --col --show
./wavyQuick.py -sat s3a -reg mwam4 -sd 2020100100 -ed 2020101000 --show
./wavyQuick.py -sat s3a -reg mwam4 -sd 2020100100 -ed 2020101000 -dump ./test.nc
""",
formatter_class=RawTextHelpFormatter)
parser.add_argument("-reg", metavar='region', help="region to check")
parser.add_argument("-sat",
metavar='satellite',
help="satellite mission, currently available\n \
\ncmems_L3_NRT:\
\n s3a - Sentinel-3A\
\n s3b - Sentinel-3B\
\n j3 - Jason-3 (reference mission)\
\n c2 - Cryosat-2\
\n al - SARAL/AltiKa\
\n cfo - CFOSAT\
\n h2b - HaiYang-2B\
\n\
\ncmems_L3_s6a:\
\n s6a - Sentinel-6A Michael Freilich\
\n\
\neumetsat_L2:\
\n s3a - Sentinel-3A\
\n s3b - Sentinel-3B\
\n\
\ncci_L2P:\
\n j1 - Jason-1\
\n j2 - Jason-2\
\n j3 - Jason-3\
\n c2 - Cryosat-2\
\n envisat - Envisat\
\n ers1 - European Remote-Sensing Satellite-1\
\n ers2 - European Remote-Sensing Satellite-2\
\n topex - TOPEX/Poseidon\
\n al - SARAL/AltiKa\
\n gfo - GEOSAT Follow-On\
\n\
\ncci_L3:\
\n multi - multimission product 1991-2018\
\n\
\nother options are:\
\n all - for all cmems satellites\
\n list - a list of chosen satellites using -l\n \
\n")
parser.add_argument("-product",
metavar='product',
help="available products as specified in *_specs.yaml")
parser.add_argument('-l',
metavar='satellite list',
help='delimited list input for sats',
type=str)
parser.add_argument("-sd",
metavar='startdate',
help="start date of time period to check")
parser.add_argument("-ed",
metavar='enddate',
help="end date of time period to check")
parser.add_argument("-mod", metavar='model', help="chosen wave model")
parser.add_argument("-var",
metavar='varalias',
help="alias for chosen variable")
parser.add_argument("-lt",
metavar='lead time',
type=int,
help="lead time from initialization")
parser.add_argument("-twin",
metavar='time window',
type=int,
help="time window for collocation")
parser.add_argument("-dist",
metavar='distance limit',
type=int,
help="distance limit for collocation")
parser.add_argument("--col",
metavar="collocation",
help="collocation",
action='store_const',
const=True)
parser.add_argument("--show",
help="show figure",
action='store_const',
const=True)
parser.add_argument("-savep",
metavar="savepath",
help="save figure to path")
parser.add_argument("-dump",
metavar="outpath",
help="dump data to .nc-file")
args = parser.parse_args()
print("Parsed arguments: ", args)
flatten = lambda l: [item for sublist in l for item in sublist]
# setup
if args.var is None:
args.var = 'Hs'
sdate = datetime(int(args.sd[0:4]), int(args.sd[4:6]), int(args.sd[6:8]),
int(args.sd[8:10]))
if args.product is None:
args.product = 'cmems_L3_NRT'
if args.twin is None:
args.twin = 30
if args.dist is None:
args.dist = 10
if args.ed is None:
edate = sdate
else:
edate = datetime(int(args.ed[0:4]), int(args.ed[4:6]),
int(args.ed[6:8]), int(args.ed[8:10]))
twin = 0
if (args.reg is None and args.mod is not None):
args.reg = args.mod
# get data
if args.sat == 'all':
satlist = ['s3a', 's3b', 'j3', 'c2', 'al', 'cfo', 'h2b']
lats = []
lons = []
var = []
time = []
dtime = []
sats = []
satnamelst = []
for sat in satlist:
try:
sa_obj_tmp = sa(sdate=sdate,
mission=sat,
edate=edate,
twin=args.twin,
region=args.reg,
varalias=args.var,
product=args.product)
if ('vars' in vars(sa_obj_tmp).keys()
and len(sa_obj_tmp.vars['time']) > 0):
sa_obj = sa_obj_tmp
lats.append(sa_obj.vars['latitude'])
lons.append(sa_obj.vars['longitude'])
var.append(sa_obj.vars[variable_info[args.var]\
['standard_name']])
time.append(sa_obj.vars['time'])
dtime.append(sa_obj.vars['datetime'])
sats.append(sat)
satnamelst.append([sat] * len(sa_obj.vars['time']))
except Exception as e:
print(e)
print(sat + ' not available')
pass
lats = flatten(lats)
lons = flatten(lons)
var = flatten(var)
time = flatten(time)
dtime = flatten(dtime)
satnames = flatten(satnamelst)
sa_obj.vars['latitude'] = np.array(lats)
sa_obj.vars['longitude'] = np.array(lons)
sa_obj.vars[variable_info[args.var]['standard_name']] = np.array(var)
sa_obj.vars['time'] = time
sa_obj.vars['datetime'] = dtime
sa_obj.region = args.reg
sa_obj.mission = str(sats)
sa_obj.mission_ts = satnames
elif args.sat == 'list':
satlist = args.l.split(',')
lats = []
lons = []
var = []
time = []
sats = []
satnamelst = []
for sat in satlist:
try:
sa_obj_tmp = sa(sdate=sdate,
mission=sat,
edate=edate,
twin=args.twin,
region=args.reg,
varalias=args.var,
product=args.product)
if ('vars' in vars(sa_obj_tmp).keys()
and len(sa_obj_tmp.vars['time']) > 0):
sa_obj = sa_obj_tmp
lats.append(sa_obj.vars['latitude'])
lons.append(sa_obj.vars['longitude'])
var.append(sa_obj.vars[variable_info[args.var]\
['standard_name']])
time.append(sa_obj.vars['time'])
sats.append(sat)
satnamelst.append([sat] * len(sa_obj.vars['time']))
except Exception as e:
print(e)
print(sat + ' not available')
pass
lats = flatten(lats)
lons = flatten(lons)
var = flatten(var)
time = flatten(time)
satnames = flatten(satnamelst)
sa_obj.vars['time'] = time
sa_obj.vars['latitude'] = np.array(lats)
sa_obj.vars['longitude'] = np.array(lons)
sa_obj.vars[variable_info[args.var]['standard_name']] = np.array(var)
sa_obj.region = args.reg
sa_obj.mission = str(sats)
sa_obj.mission_ts = satnames
else:
sa_obj = sa(sdate=sdate,
mission=args.sat,
edate=edate,
twin=args.twin,
region=args.reg,
varalias=args.var,
product=args.product)
# plot
if (args.mod is None and sa_obj.region not in model_dict):
plot_sat(sa_obj, savepath=args.savep, showfig=args.show)
elif (args.mod is None and sa_obj.region in model_dict):
print('Chosen region is a specified model domain')
mc_obj = mc(model=sa_obj.region,
fc_date=model_dict[sa_obj.region]['grid_date'],
varalias=args.var)
plot_sat(sa_obj, mc_obj=mc_obj, savepath=args.savep, showfig=args.show)
elif (args.mod is not None and args.col is True and sdate == edate):
# get model collocated values
mc_obj = mc(model=args.mod,
fc_date=edate,
leadtime=args.lt,
varalias=args.var)
#collocation
coll_obj = coll(mc_obj_in=mc_obj,
obs_obj_in=sa_obj,
distlim=args.dist,
leadtime=args.lt)
valid_dict = validate(coll_obj.vars)
disp_validation(valid_dict)
comp_fig(sa_obj=sa_obj,
mc_obj=mc_obj,
coll_obj=coll_obj,
savepath=args.savep,
showfig=args.show)
elif (args.mod is not None and args.col is False):
# get model collocated values
mc_obj = mc(model=sa_obj.region,
fc_date=edate,
leadtime=args.lt,
varalias=args.var)
results_dict = {'valid_date':[edate],
'date_matches':[edate-timedelta(minutes=args.twin),
edate+timedelta(minutes=args.twin)],
'model_lons_matches':sa_obj.vars['longitude'],
'model_lats_matches':sa_obj.vars['latitude'],
'sat_matches':sa_obj.vars[variable_info[args.var]\
['standard_name']]}
comp_fig(args.mod,
sa_obj,
mc_obj.vars[variable_info[args.var]['standard_name']],
mc_obj.vars['longitude'],
mc_obj.vars['latitude'],
results_dict,
variable_info[args.var]['standard_name'],
savepath=args.savep,
showfig=args.show)
elif (args.mod is not None and args.col is True and args.sat is not None):
#collocation
date_incr = model_dict[args.mod].get('date_incr', 1)
coll_obj = coll(model=args.mod,
obs_obj_in=sa_obj,
distlim=args.dist,
leadtime=args.lt,
date_incr=date_incr)
valid_dict = coll_obj.validate_collocated_values()
if args.show is True:
print(
"Figure not shown! Request is ambiguous for multiple model time steps."
)
# dump to .ncfile
if args.dump is not None:
if args.col is not None:
coll_obj.write_to_nc(pathtofile=args.dump)
else:
sa_obj.write_to_nc(pathtofile=args.dump)
import zipfile
import tempfile
from tqdm import tqdm
import xarray as xr
from datetime import timedelta
import netCDF4
# own imports
from wavy.ncmod import ncdumpMeta, get_filevarname
from wavy.ncmod import read_netcdfs, read_netcdfs_zipped_lru
from wavy.wconfig import load_or_default
from wavy.utils import parse_date
# ---------------------------------------------------------------------#
# read yaml config files:
satellite_dict = load_or_default('satellite_specs.yaml')
variable_info = load_or_default('variable_info.yaml')
def unzip_eumetsat(pathlst, tmpdir):
"""
Function to unzip eumetsat files prior to reading
param:
pathlst - list of paths to zipped files
tmpdir - temporary folder to unzipped files
return:
pathlst_new - new list of paths to unzipped files
"""
for count, f in enumerate(pathlst):
