def make_frost_reference_time_period(sdate,edate):
sdate = parse_date(sdate)
edate = parse_date(edate)
formatstr = '%Y-%m-%dT%H:%M:00.000Z'
refstr = '{}/{}'.format(sdate.strftime(formatstr),
edate.strftime(formatstr))
return refstr
def quicklook(self,m=True,projection=None,date=None,**kwargs):
if m is True:
import cartopy.crs as ccrs
import cmocean
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1.inset_locator import inset_axes
dt = parse_date(date)
lons = self.vars['longitude']
lats = self.vars['latitude']
var = self.vars[self.stdvarname]
fc_date = self.fc_date
if date is not None:
# find date in self.vars and give a snapshot
idx = self.vars['datetime'].index(dt)
var = self.vars[self.stdvarname][idx,:]
fc_date = self.fc_date[idx]
elif (date is None and len(var.shape)==3):
# find date in self.vars and give a snapshot
var = self.vars[self.stdvarname][0,:]
fc_date = self.fc_date[0]
if projection is None:
projection = ccrs.PlateCarree()
# parse kwargs
cflevels = kwargs.get('cflevels',10)
clevels = kwargs.get('clevels',10)
if kwargs.get('cmap') is None:
cmap = cmocean.cm.amp
else:
cmap = kwargs.get('cmap')
lonmax,lonmin = np.max(lons),np.min(lons)
latmax,latmin = np.max(lats),np.min(lats)
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1, projection=projection)
ax.set_extent( [lonmin, lonmax,latmin, latmax],
crs = projection )
cf = ax.contourf(lons,lats, var,
cmap=cmap,levels=cflevels,
transform=ccrs.PlateCarree())
c = ax.contour(lons,lats, var,
cmap=cmap,levels=clevels,
transform=ccrs.PlateCarree())
axins = inset_axes(ax,
width="5%", # width = 5% of parent_bbox width
height="100%", # height : 50%
loc='lower left',
bbox_to_anchor=(1.01, 0., 1, 1),
bbox_transform=ax.transAxes,
borderpad=0,
)
fig.colorbar(cf, cax=axins, label=self.varalias
+ ' [' + self.units + ']')
ax.coastlines()
ax.gridlines()
plt.subplots_adjust(bottom=0.1, right=0.8, top=0.9)
ax.set_title(self.model + ' for ' + str(fc_date))
plt.show()
def get_frost_df_v1(r,nID,sensor,varalias):
# base df
df = pd.json_normalize(r.json()['data']['tseries'])
# df to be concatenated initialized with time
dfc = pd.json_normalize(r.json()
['data']['tseries'][0]['observations'])['time'].to_frame()
stdvarname = variable_info[varalias]['standard_name']
sensoridx = insitu_dict[nID]['sensor'][sensor]
idx = df['header.extra.element.id']\
[df['header.extra.element.id']==stdvarname]\
.index.to_list()
idxs = df['header.id.sensor']\
[df['header.id.sensor']==sensoridx]\
.index.to_list()
dftmp = pd.json_normalize(r.json()\
['data']['tseries'][idxs[sensoridx]]['observations'])\
['body.data'].to_frame()
dftmp = dftmp.rename(columns={ dftmp.columns[0]: stdvarname })
dftmp[stdvarname] = dftmp[stdvarname].mask(dftmp[stdvarname] < 0, np.nan)
dfc = pd.concat([dfc, dftmp.reindex(dfc.index)], axis=1)
var = list(dfc[stdvarname].values)
timedt = [parse_date(d) for d in dfc['time'].values]
timedt = [dt.replace(tzinfo=None) for dt in timedt]
time = netCDF4.date2num(timedt,variable_info['time']['units'])
sensor = list(insitu_dict[nID]['sensor'].keys())[0]
lons = [insitu_dict[nID]['coords'][sensor]['lon']]*len(var)
lats = [insitu_dict[nID]['coords'][sensor]['lat']]*len(var)
vardict = {
stdvarname:var,
'time':time,
'datetime':timedt,
'time_unit':variable_info['time']['units'],
'longitude':lons,
'latitude':lats
}
#return dfc
return vardict
def __init__(self,
model='mwam4',
sdate=None,
edate=None,
date_incr=1,
fc_date=None,
leadtime=None,
varalias='Hs',
st_obj=None,
transform_lons=None):
print('# ----- ')
print(" ### Initializing model_class object ###")
print(' ')
# parse and translate date input
sdate = parse_date(sdate)
edate = parse_date(edate)
fc_date = parse_date(fc_date)
if st_obj is not None:
sdate = st_obj.sdate
edate = st_obj.edate
varalias = st_obj.varalias
if fc_date is not None:
print("Requested time: ", str(fc_date))
elif (edate is None and fc_date is None and sdate is not None):
fc_date = sdate
print("Requested time: ", str(fc_date))
elif (sdate is None and edate is None and fc_date is None):
now = datetime.now()
fc_date = datetime(now.year, now.month, now.day, now.hour)
print("Requested time: ", str(fc_date))
elif (sdate is not None and edate is not None
and date_incr is not None):
fc_date = sdate
print("Requested time frame: " + str(sdate) + " - " + str(edate))
print(" ")
print(" ## Read files ...")
t0 = time.time()
vardict, \
fc_date, leadtime, \
filestr, \
filevarname = get_model(model=model,sdate=sdate,edate=edate,
date_incr=date_incr,fc_date=fc_date,
leadtime=leadtime,varalias=varalias,
st_obj=st_obj,transform_lons=transform_lons)
stdname = variable_info[varalias]['standard_name']
units = variable_info[varalias]['units']
varname = filevarname
# define class variables
self.fc_date = fc_date
self.sdate = sdate
self.edate = edate
self.leadtime = leadtime
self.model = model
self.varalias = varalias
self.varname = varname
self.stdvarname = stdname
self.units = units
self.vars = vardict
self.filestr = filestr
t1 = time.time()
print(" ")
print( '## Summary:')
print("Time used for retrieving model data:", round(t1 - t0, 2),
"seconds")
print(" ")
print(" ### model_class object initialized ###")
print('# ----- ')
def get_model(model=None,
sdate=None,
edate=None,
date_incr=None,
fc_date=None,
leadtime=None,
varalias=None,
st_obj=None,
transform_lons=None):
"""
toplevel function to get model data
"""
if st_obj is not None:
sdate = st_obj.sdate
edate = st_obj.edate
varalias = st_obj.varalias
if (sdate is not None
and edate is not None
and date_incr is not None):
fc_date = make_fc_dates(sdate, edate, date_incr)
filestr = make_model_filename_wrapper(model=model,
fc_date=fc_date,
leadtime=leadtime)
if (isinstance(filestr, list) and st_obj is None):
vardict, \
filevarname = get_model_fc_mode(filestr=filestr[0],model=model,
fc_date=fc_date[0],varalias=varalias)
vardict[variable_info[varalias]['standard_name']]=\
[vardict[variable_info[varalias]['standard_name']]]
vardict['time'] = [vardict['time']]
vardict['datetime'] = [vardict['datetime']]
vardict['leadtime'] = leadtime
for i in tqdm(range(1, len(filestr))):
tmpdict, \
filevarname = get_model_fc_mode(\
filestr=filestr[i],model=model,
fc_date=fc_date[i],varalias=varalias)
vardict[variable_info[varalias]['standard_name']].append(
tmpdict[variable_info[varalias]['standard_name']])
vardict['time'].append(tmpdict['time'])
vardict['datetime'].append(tmpdict['datetime'])
vardict[variable_info[varalias]['standard_name']]=\
np.array(vardict[variable_info[varalias]['standard_name']])
else:
vardict, \
filevarname = get_model_fc_mode(filestr=filestr,model=model,
fc_date=fc_date,varalias=varalias)
vardict['time'] = [vardict['time']]
vardict['datetime'] = [vardict['datetime']]
vardict['leadtime'] = leadtime
# transform to datetime
with NoStdStreams():
tmpd = [parse_date(str(d)) for d in vardict['datetime']]
vardict['datetime'] = tmpd
del tmpd
if transform_lons==180:
# transform longitudes from -180 to 180
vardict['longitude'] = ((vardict['longitude'] - 180) % 360) - 180
elif transform_lons==360:
print('not yet implemented !!')
return vardict, fc_date, leadtime, filestr, filevarname
def __init__(self,
mc_obj_in=None,
obs_obj_in=None,
poi=None,
col_obj_in=None,
model=None,
distlim=None,
leadtime=None,
date_incr=1,
varalias='Hs',
twin=30):
print('# ----- ')
print(" ### Initializing collocation_class object ###")
print(" ")
# make clones to prevent overwriting
mc_obj = deepcopy(mc_obj_in)
obs_obj = deepcopy(obs_obj_in)
col_obj = deepcopy(col_obj_in)
self.varalias = varalias
if mc_obj is not None:
model = mc_obj.model
self.model = model
if isinstance(obs_obj, satellite_class):
self.obsname = obs_obj.mission
self.mission = obs_obj.mission
self.obstype = "satellite_altimeter"
self.stdvarname = obs_obj.stdvarname
self.region = obs_obj.region
self.sdate = obs_obj.sdate
self.edate = obs_obj.edate
if isinstance(obs_obj, insitu_class):
obs_obj.twin = insitu_dict[obs_obj.nID].get('twin', None)
self.obsname = obs_obj.nID + '_' + obs_obj.sensor
self.obstype = 'insitu'
self.nID = obs_obj.nID
self.sensor = obs_obj.sensor
self.stdvarname = obs_obj.stdvarname
self.units = variable_info[self.varalias].get('units')
self.sdate = obs_obj.sdate
self.edate = obs_obj.edate
if poi is not None:
# poi is of type dict
self.nID = poi['nID']
self.obsname = poi['nID']
self.obstype = 'POI'
stdvarname = variable_info[varalias]['standard_name']
self.stdvarname = stdvarname
self.units = variable_info[varalias].get('units')
self.sdate = parse_date(poi['time'][0])
self.edate = parse_date(poi['time'][-1])
# define class variables
self.leadtime = leadtime
if leadtime is None:
self.leadtime = 'best'
self.leadtimestr = 'best'
elif isinstance(self.leadtime, str):
self.leadtime = leadtime
leadtimestr = leadtime
self.leadtimestr = leadtime
else:
leadtimestr = "{:0>3d}".format(self.leadtime)
self.leadtime = leadtime
self.leadtimestr = leadtimestr
# get vars dictionary
print(" ")
print(" ## Collocate ... ")
# for t in range(1):
try:
t0 = time.time()
results_dict = collocate(mc_obj=mc_obj,
obs_obj=obs_obj,
col_obj=col_obj,
poi=poi,
model=model,
distlim=distlim,
leadtime=self.leadtime,
date_incr=date_incr,
varalias=self.varalias)
self.vars = results_dict
self.fc_date = results_dict['datetime']
t1 = time.time()
print(" ")
print(" ## Summary:")
print(len(self.vars['time']), " values collocated.")
print("Time used for collocation:", round(t1 - t0, 2), "seconds")
print(" ")
print(" ### Collocation_class object initialized ###")
except Exception as e:
print(e)
self.error = e
print("! No collocation_class object initialized !")
# add class variables
print('# ----- ')
def collocate_poi_ts(indict,model=None,distlim=None,\
leadtime=None,date_incr=None,varalias=None,twin=None):
"""
indict: mandatory - lons, lats, time, values
optional - leadtime, distlim, date_incr
"""
# get stdvarname
stdvarname = variable_info[varalias]['standard_name']
# datetime or str
if isinstance(indict['time'][0], str):
poi_dtimes = [parse_date(t) for t in indict['time']]
elif isinstance(indict['time'][0], datetime):
poi_dtimes = indict['time']
else:
print('no valid time/datetime format for poi')
print('use either str or datetime')
fc_date = make_fc_dates(poi_dtimes[0], poi_dtimes[-1], date_incr)
# get coinciding date between fc_date and dates in obs_obj
idx1 = collocate_times(unfiltered_t=poi_dtimes,
target_t=fc_date,
twin=twin)
# find valid/coinciding fc_dates
if len(idx1) > len(fc_date):
print('Muliple assignments within given time window')
print('--> only closest to time stamp is chosen')
idx_closest = get_closest_date(\
list(np.array(poi_dtimes)[idx1]),\
fc_date)
idx1 = list(np.array(idx1)[idx_closest])
# adjust obs_obj according to valid dates
for key in indict.keys():
if (key != 'time_unit' and key != 'meta' and key != 'nID'):
indict[key] = list(np.array(indict[key])[idx1])
poi_dtimes = list(np.array(poi_dtimes)[idx1])
del idx1
# find valid dates for given leadtime and model
fc_date = find_valid_fc_dates_for_model_and_leadtime(\
fc_date,model,leadtime)
# adjust fc_date according to obs date
idx2 = collocate_times(unfiltered_t=fc_date,
target_t=poi_dtimes,
twin=twin)
fc_date = list(np.array(fc_date)[idx2])
del idx2
# compute time based on time unit from variable definition
time_unit = variable_info['time']['units']
time = netCDF4.date2num(poi_dtimes, time_unit)
# check if file exists and if it includes desired time and append
model_vals = []
model_lons = []
model_lats = []
obs_vals = []
obs_lons = []
obs_lats = []
collocation_idx_x = []
collocation_idx_y = []
distance = []
time_lst = []
dtimes = []
switch = 0
for d in tqdm(range(len(fc_date))):
# for t in range(1):
with NoStdStreams():
check = False
check = check_if_file_is_valid(fc_date[d], model, leadtime)
if check == True:
# retrieve model
fname = make_model_filename_wrapper(model, fc_date[d],
leadtime)
# get hold of variable names (done only once)
if switch == 0:
meta = ncdumpMeta(fname)
lonsname = get_filevarname('lons', variable_info,
model_dict[model], meta)
latsname = get_filevarname('lats', variable_info,
model_dict[model], meta)
timename = get_filevarname('time', variable_info,
model_dict[model], meta)
filevarname = get_filevarname(varalias, variable_info,
model_dict[model], meta)
mlons = xr.open_dataset(fname)[lonsname].values
# secure lons from -180 to 180
mlons = ((mlons - 180) % 360) - 180
mlats = xr.open_dataset(fname)[latsname].values
# ensure matching dimension
if len(mlons.shape) == 1:
Mlons, Mlats = np.meshgrid(mlons, mlats)
else:
Mlons, Mlats = mlons, mlats
switch = 1
plon = [indict['longitude'][d]]
plat = [indict['latitude'][d]]
index_array_2d, distance_array, _ = \
collocation_fct(plon,plat,Mlons,Mlats)
dst = xr.open_dataset(fname)[timename].values
tidx = list(dst).index(np.datetime64(fc_date[d]))
# impose distlim
if distance_array[0] < distlim * 1000:
idx_x = index_array_2d[0][0]
idx_y = index_array_2d[1][0]
model_lons.append(Mlons[idx_x, idx_y])
model_lats.append(Mlats[idx_x, idx_y])
vals = xr.open_dataset(fname)[filevarname]\
[tidx,idx_x,idx_y].values
model_vals.append(vals.item())
obs_vals.append(indict['obs'][d])
obs_lons.append(indict['longitude'][d])
obs_lats.append(indict['latitude'][d])
collocation_idx_x.append(idx_x)
collocation_idx_y.append(idx_y)
distance.append(distance_array[0])
time_lst.append(time[d])
dtimes.append(poi_dtimes[d])
results_dict = {
'valid_date': dtimes,
'time': time_lst,
'time_unit': time_unit,
'datetime': dtimes,
'distance': distance,
'model_values': model_vals,
'model_lons': model_lons,
'model_lats': model_lats,
'obs_values': obs_vals,
'obs_lons': obs_lons,
'obs_lats': obs_lats,
'collocation_idx_x': collocation_idx_x,
'collocation_idx_y': collocation_idx_y
}
return results_dict
def __init__(self,
sdate=None,
mission='s3a',
product='cmems_L3_NRT',
edate=None,
twin=30,
download=False,
path_local=None,
region='mwam4',
nproc=1,
varalias='Hs',
api_url=None,
filterData=False,
poi=None,
distlim=None,
**kwargs):
print('# ----- ')
print(" ### Initializing satellite_class object ###")
print(" ")
# parse and translate date input
sdate = parse_date(sdate)
edate = parse_date(edate)
# check settings
if (sdate is None and edate is None and poi is not None):
sdate = poi['datetime'][0]
edate = poi['datetime'][-1]
elif (edate is None and sdate is not None):
print("Requested time: ", str(sdate))
edate = sdate
elif (edate is None and sdate is None):
now = datetime.now()
sdate = datetime(now.year, now.month, now.day, now.hour)
edate = sdate
print("Requested time: ", str(sdate))
else:
print("Requested time frame: " + str(sdate) + " - " + str(edate))
stdname = variable_info[varalias].get('standard_name')
units = variable_info[varalias].get('units')
# define some class variables
self.sdate = sdate
self.edate = edate
self.varalias = varalias
self.units = units
self.stdvarname = stdname
self.twin = twin
self.region = region
self.mission = mission
self.obstype = 'satellite_altimeter'
self.product = product
self.provider = satellite_dict[product].get('provider')
self.processing_level = \
satellite_dict[product].get('processing_level')
print('Chosen time window is:', twin, 'min')
# make satpaths
if path_local is None:
path_template = satellite_dict[product]\
['dst']\
['path_template']
self.path_local = path_template
else:
self.path_local = path_local
# retrieve files
if download is False:
print("No download initialized, checking local files")
else:
print("Downloading necessary files ...")
get_remote_files(\
path_local=path_local,
sdate=sdate,edate=edate,
twin=twin,nproc=nproc,
product=product,
api_url=api_url,
mission=mission,
dict_for_sub=vars(self))
print(" ")
print(" ## Find files ...")
t0 = time.time()
pathlst, _ = get_local_files(sdate,
edate,
twin,
product,
vars(self),
path_local=path_local)
print(" ")
print(" ## Read files ...")
if len(pathlst) > 0:
# for i in range(1):
try:
if filterData == True:
# extend time period due to filter
if 'stwin' not in kwargs.keys():
kwargs['stwin'] = 1 # needs to be changed
if 'etwin' not in kwargs.keys():
kwargs['etwin'] = 1
twin_tmp = twin + kwargs['stwin'] + kwargs['etwin']
# retrieve data
rvardict = get_sat_ts(sdate, edate, twin_tmp, region,
product, pathlst, varalias, poi,
distlim)
# filter data
rvardict = filter_main(rvardict,
varalias=varalias,
**kwargs)
# crop to original time period
sdate_tmp = sdate - timedelta(minutes=twin)
edate_tmp = sdate + timedelta(minutes=twin)
rvardict = crop_vardict_to_period(rvardict, sdate_tmp,
edate_tmp)
self.filter = True
self.filterSpecs = kwargs
else:
rvardict = get_sat_ts(sdate, edate, twin, region, product,
pathlst, varalias, poi, distlim)
# make ts in vardict unique
rvardict = vardict_unique(rvardict)
# rm NaNs
rvardict = rm_nan_from_vardict(varalias, rvardict)
# find variable name as defined in file
if (product == 'cmems_L3_NRT' or product == 'cmems_L3_MY'
or product == 'cmems_L3_s6a'):
ncdict = ncdumpMeta(pathlst[0])
elif (product == 'cci_L2P' or product == 'cci_L3'):
ncdict = ncdumpMeta(pathlst[0])
elif product == 'eumetsat_L2':
tmpdir = tempfile.TemporaryDirectory()
zipped = zipfile.ZipFile(pathlst[0])
enhanced_measurement = zipped.namelist()[-1]
extracted = zipped.extract(enhanced_measurement,
path=tmpdir.name)
ncdict = ncdumpMeta(extracted)
tmpdir.cleanup()
with NoStdStreams():
filevarname = get_filevarname(varalias, variable_info,
satellite_dict[product],
ncdict)
rvardict['meta'] = ncdict
# define more class variables
self.vars = rvardict
self.varname = filevarname
t1 = time.time()
print(" ")
print('## Summary:')
print(str(len(self.vars['time'])) + " footprints retrieved.")
print("Time used for retrieving satellite data:",\
round(t1-t0,2),"seconds")
print(" ")
print("### Satellite object initialized ###")
print('# ----- ')
except Exception as e:
print(e)
print('Error encountered')
print('No satellite_class object initialized')
else:
print('No satellite data found')
print('No satellite_class object initialized')
print('# ----- ')
def __init__(self,nID,sensor,sdate,edate,varalias='Hs',
filterData=False,**kwargs):
# parse and translate date input
sdate = parse_date(sdate)
edate = parse_date(edate)
print('# ----- ')
print(" ### Initializing insitu_class object ###")
print(" ")
print ('Chosen period: ' + str(sdate) + ' - ' + str(edate))
stdvarname = variable_info[varalias]['standard_name']
# for i in range(1):
try:
self.stdvarname = stdvarname
self.varalias = varalias
self.units = variable_info[varalias].get('units')
self.sdate = sdate
self.edate = edate
self.nID = nID
self.sensor = sensor
self.obstype = 'insitu'
if ('tags' in insitu_dict[nID].keys() and
len(insitu_dict[nID]['tags'])>0):
self.tags = insitu_dict[nID]['tags']
print(" ")
print(" ## Read files ...")
t0=time.time()
if filterData == False:
vardict, fifo, pathtofile = \
get_insitu_ts(\
nID = nID,
sensor = sensor,
sdate = sdate,
edate = edate,
varalias = varalias,
basedate = self.basedate,
dict_for_sub = vars(self),
**kwargs)
elif filterData == True:
# determine start and end date
if 'stwin' not in kwargs.keys():
kwargs['stwin'] = 3
if 'etwin' not in kwargs.keys():
kwargs['etwin'] = 0
sdate_new = sdate - timedelta(hours=kwargs['stwin'])
edate_new = edate + timedelta(hours=kwargs['etwin'])
tmp_vardict, fifo, pathtofile = \
get_insitu_ts(
nID = nID,
sensor = sensor,
sdate = sdate_new,
edate = edate_new,
varalias = varalias,
basedate = self.basedate,
dict_for_sub = vars(self),
**kwargs)
vardict = filter_main(tmp_vardict,
varalias=varalias,
**kwargs)
# cut to original sdate and edate
time_cut = np.array(vardict['time'])[ \
( (np.array(vardict['datetime'])>=sdate)
& (np.array(vardict['datetime'])<=edate)
) ]
var_cut = np.array(vardict[stdvarname])[ \
( (np.array(vardict['datetime'])>=sdate)
& (np.array(vardict['datetime'])<=edate)
) ]
lon_cut = np.array(vardict['longitude'])[ \
( (np.array(vardict['datetime'])>=sdate)
& (np.array(vardict['datetime'])<=edate)
) ]
lat_cut = np.array(vardict['latitude'])[ \
( (np.array(vardict['datetime'])>=sdate)
& (np.array(vardict['datetime'])<=edate)
) ]
dtime_cut = np.array(vardict['datetime'])[ \
( (np.array(vardict['datetime'])>=sdate)
& (np.array(vardict['datetime'])<=edate)) ]
vardict['time'] = list(time_cut)
vardict['datetime'] = list(dtime_cut)
vardict[stdvarname] = list(var_cut)
vardict['longitude'] = list(lon_cut)
vardict['latitude'] = list(lat_cut)
self.filter = True
self.filterSpecs = kwargs
self.vars = vardict
self.lat = np.nanmean(vardict['latitude'])
self.lon = np.nanmean(vardict['longitude'])
if fifo == 'nc':
print(pathtofile)
meta = ncdumpMeta(sdate.strftime(pathtofile))
self.vars['meta'] = meta
varname = get_filevarname(varalias,
variable_info,
insitu_dict[nID],
meta)
self.varname = varname
else:
self.varname = varalias
if fifo == 'frost':
self.sensor = sensor
t1=time.time()
print(" ")
print( '## Summary:')
print(str(len(self.vars['time'])) + " values retrieved.")
print("Time used for retrieving insitu data:",\
round(t1-t0,2),"seconds")
print(" ")
print (" ### insitu_class object initialized ### ")
except Exception as e:
print(e)
self.error = e
print ("! No insitu_class object initialized !")
print ('# ----- ')