def align_synoptic_class_with_pw(path):
import xarray as xr
from aux_gps import dim_intersection
from aux_gps import save_ncfile
from aux_gps import xr_reindex_with_date_range
pw = xr.load_dataset(path / 'GNSS_PW_thresh_50_homogenized.nc')
pw = pw[[x for x in pw if '_error' not in x]]
syn = read_synoptic_classification(report=False).to_xarray()
# syn = syn.drop(['Name-EN', 'Name-HE'])
syn = syn['class']
syn = syn.sel(time=slice('1996', None))
syn = syn.resample(time='5T').ffill()
ds_list = []
for sta in pw:
print('aligning station {} with synoptics'.format(sta))
new_time = dim_intersection([pw[sta], syn])
syn_da = xr.DataArray(syn.sel(time=new_time))
syn_da.name = '{}_class'.format(sta)
syn_da = xr_reindex_with_date_range(syn_da)
ds_list.append(syn_da)
ds = xr.merge(ds_list)
ds = ds.astype('int8')
ds = ds.fillna(0)
filename = 'GNSS_synoptic_class.nc'
save_ncfile(ds, path, filename)
return ds
def post_process_ims_stations(ds, window, savepath, gis_path, dem_path,
axis_path):
"""fill TD with hourly mean if NaN and smooth, then fill in station_lat
and lon and alt from DEM, finally interpolate to AXIS coords and save"""
from aux_gps import fill_na_xarray_time_series_with_its_group
from ims_procedures import analyse_10mins_ims_field
from axis_process import produce_rinex_filenames_at_time_window
from ims_procedures import IMS_interpolating_to_GNSS_stations_israel
from aux_gps import save_ncfile
import pandas as pd
now_dt = pd.Timestamp.utcnow().floor('H')
ds = fill_na_xarray_time_series_with_its_group(ds, grp='hour')
ds = analyse_10mins_ims_field(ds=ds, var='TD', gis_path=gis_path,
dem_path=dem_path)
ds_axis = IMS_interpolating_to_GNSS_stations_israel(
dt=None, start_year=str(now_dt.year), verbose=True, savepath=None,
network='axis', ds_td=ds, cut_days_ago=None, axis_path=axis_path)
now_dt = pd.Timestamp.utcnow().floor('H')
names = produce_rinex_filenames_at_time_window(end_dt=now_dt,
window=window)
st_str = names[0][4:8]
end_str = names[-1][4:8]
filename = 'AXIS_TD_{}-{}.nc'.format(st_str, end_str)
save_ncfile(ds_axis, savepath, filename)
return ds_axis
def produce_final_dsea_pwv(ims_station=None, savepath=None, use_pressure=True):
"""use ims_station='SEDOM' to get close ts, pressure"""
import xarray as xr
from aux_gps import save_ncfile
if ims_station is not None:
ts = xr.open_dataset(ims_path /
'IMS_TD_israeli_10mins.nc')[ims_station]
pres = xr.open_dataset(ims_path /
'IMS_BP_israeli_10mins.nc')[ims_station]
ts.load()
pres.load()
ts = ts.resample(time='5T').ffill()
pres = pres.resample(time='5T', keep_attrs=True).ffill()
if use_pressure:
wetz = produce_wetz_dsea_from_ztd(pres=pres)
else:
p = dsea_gipsy_path / 'results-{}'.format('GPT2')
wetz = read_all_final_tdps_dsea(return_mean=True, dryz=False)
pwv = produce_pwv_from_zwd_with_ts_tm_from_deserve(ts=ts, zwd=wetz)
pwv.attrs['units'] = 'mm'
pwv.attrs['long_name'] = 'precipitable water vapor'
pwv.name = 'pwv'
pwv.attrs[
'action'] = 'corrected wetz using surface pressure and ts-tm from radiosonde'
if savepath is not None:
if ims_station is not None:
filename = 'DSEA_PWV_{}.nc'.format(ims_station)
save_ncfile(pwv, savepath, filename)
return pwv
def read_BD_ceilometer_yoav_all_years(path=ceil_path, savepath=None):
from aux_gps import path_glob
from aux_gps import save_ncfile
import pandas as pd
files = path_glob(path, 'ceilometer_BD*.csv')
dfs = []
for file in files:
dfs.append(read_BD_ceilometer_yoav_one_year_csv(file))
df = pd.concat(dfs)
df = df.sort_index()
names = [x.split('[')[0] for x in df.columns]
units = [x.split('[')[1].split(']')[0] for x in df.columns]
long_names = [
'total cloud cover', 'cloud cover of the most cloudy layer',
'cloud cover of the 1st cloud layer', '1st cloud base height',
'cloud cover of the 2nd cloud layer', '2nd cloud base height',
'cloud cover of the 3rd cloud layer', '3rd cloud base height',
'cloud cover of the 4th cloud layer', '4th cloud base height',
'cloud cover of the 5th cloud layer', '5th cloud base height',
'Mixing layer height'
]
df.columns = names
# fix cloud height to meters again for until 22-09-2013:
hs = [x for x in df.columns if '_H' in x]
df.loc[:'2013-09-22', hs] *= (1 / 0.3048)
ds = df.to_xarray()
for i, da in enumerate(ds):
ds[da].attrs['units'] = units[i]
ds[da].attrs['long_name'] = long_names[i]
if savepath is not None:
filename = 'BD_clouds_and_MLH_from_ceilometers.nc'
save_ncfile(ds, savepath, filename)
return ds
def read_geodetic_positions_and_height(path=jpl_path):
import pandas as pd
import requests
from io import StringIO
from aux_gps import save_ncfile
url = 'https://sideshow.jpl.nasa.gov/post/tables/table2.html'
r = requests.get(url)
data = r.text
df = pd.read_csv(StringIO(data), delim_whitespace=True, skiprows=7)
df.drop(df.tail(1).index, inplace=True) # drop last n rows
df = df.unstack()
cols0 = df.columns.get_level_values(0)
cols1 = df.columns.get_level_values(1)
cols = ['{}_{}'.format(x, y) for x, y in zip(cols0, cols1)]
df.columns = cols
df.index.name = 'station'
ds = df.to_xarray()
pos_das = [x for x in ds if 'POS' in x]
vel_das = [x for x in ds if 'VEL' in x]
for da in pos_das:
if 'V' in da.split('_')[0]:
ds[da].attrs['units'] = 'mm'
else:
ds[da].attrs['units'] = 'deg'
ds['SN_POS'].attrs['units'] = 'mm'
ds['SE_POS'].attrs['units'] = 'mm'
for da in vel_das:
ds[da].attrs['units'] = 'mm/yr'
ds.attrs['name'] = 'geodetic positions and height and velocities'
ds.attrs['reference frame'] = 'IGS14'
ds.attrs['reference epoch'] = '2020-01-01'
ds.attrs['reference ellipsoid'] = 'GRS80'
filename = 'jpl_geodetic_positions_velocities.nc'
save_ncfile(ds, path, filename)
return ds
def read_break_estimates_jpl_gipsyx_site(path=jpl_path):
import pandas as pd
import requests
from io import StringIO
from aux_gps import save_ncfile
url = 'https://sideshow.jpl.nasa.gov/post/tables/table3.html'
r = requests.get(url)
data = r.text
df = pd.read_csv(StringIO(data), delim_whitespace=True, skiprows=4)
df.drop(df.tail(1).index, inplace=True) # drop last n rows
ds = df.to_xarray()
ds = ds.rename({'level_0': 'station', 'level_1': 'year'})
for da in ds:
ds[da].attrs['units'] = 'mm'
ds.attrs['units'] = 'mm'
ds['N'].attrs['long_name'] = 'north'
ds['E'].attrs['long_name'] = 'east'
ds['V'].attrs['long_name'] = 'vertical'
ds['SN'].attrs['long_name'] = 'north error'
ds['SE'].attrs['long_name'] = 'east error'
ds['SV'].attrs['long_name'] = 'vertical error'
ds.attrs['name'] = 'break estimates'
filename = 'jpl_break_estimates.nc'
save_ncfile(ds, path, filename)
return ds
def read_multi_station_tdp_file(file, stations, savepath=None):
import pandas as pd
import xarray as xr
from aux_gps import save_ncfile
df_raw = pd.read_csv(file, header=None, delim_whitespace=True)
# first loop over list of stations and extract the data:
df_stns = [df_raw[df_raw.iloc[:, -1].str.contains(x)] for x in stations]
# now process each df from df_stns and extract the keys:
keys = ['DryZ', 'WetZ', 'GradNorth', 'GradEast', 'Pos.X', 'Pos.Y', 'Pos.Z']
desc = ['Zenith Hydrostatic Delay', 'Zenith Wet Delay',
'North Gradient of Zenith Wet Delay',
'East Gradient of Zenith Wet Delay',
'WGS84(geocentric) X coordinate',
'WGS84(geocentric) Y coordinate', 'WGS84(geocentric) Z coordinate']
units = ['cm', 'cm', 'cm/m', 'cm/m', 'm', 'm', 'm']
desc_dict = dict(zip(keys, desc))
units_dict = dict(zip(keys, units))
ppps = []
for df_stn in df_stns:
df_list = [df_stn[df_stn.iloc[:, -1].str.contains(x)] for x in keys]
# make sure that all keys in df have the same length:
# assert len(set([len(x) for x in df_list])) == 1
# translate the seconds col to datetime:
seconds = df_list[-1].iloc[:, 0]
dt = pd.to_datetime('2000-01-01T12:00:00')
time = dt + pd.to_timedelta(seconds, unit='sec')
# build a new df that contains all the vars(from keys):
ppp = pd.DataFrame(index=time)
ppp.index.name = 'time'
for i, df in enumerate(df_list):
if df.empty:
continue
df.columns = ['seconds', 'to_drop', keys[i], keys[i] + '_error',
'meta']
ppp[keys[i]] = df[keys[i]].values
ppp[keys[i] + '_error'] = df[keys[i] + '_error'].values
# rename all the Pos. to nothing:
# ppp.columns = ppp.columns.str.replace('Pos.', '')
ppps.append(ppp.to_xarray())
ds = xr.concat(ppps, 'station')
ds['station'] = stations
for da in ds:
if 'Wet' in da or 'Dry' in da or 'Grad' in da:
ds[da] = ds[da] * 100
if 'Wet' in da:
ds[da].attrs['units'] = units_dict.get('WetZ')
elif 'Grad' in da:
ds[da].attrs['units'] = units_dict.get('GradNorth')
ds[da].attrs['long_name'] = desc_dict.get(da, '')
if 'Pos' in da:
ds[da].attrs['units'] = 'm'
pos_names = [x for x in ds if 'Pos' in x]
pos_new_names = [x.split('.')[-1] for x in pos_names]
ds = ds.rename(dict(zip(pos_names, pos_new_names)))
if savepath is not None:
# filename = file.as_posix().split('/')[-1].split()
save_ncfile(ds, savepath, 'smoothFinal.nc')
return ds
def run_harmonic_analysis_on_all_jpl_products(path=jpl_path, savepath=jpl_path/'harmonic_analysis'):
from aux_gps import save_ncfile
dss = read_geodetic_positions_and_height(path=path)
for i, station in enumerate(dss['station'].values):
print('processing station {} ({} out of {})'.format(station, i+1, dss['station'].size))
ds = produce_seasonal_trend_breakdown_time_series_from_jpl_gipsyx_site(station=station, verbose=False, plot=False)
filename = '{}_V_harmonic_mm.nc'.format(station)
save_ncfile(ds, savepath, filename)
return
def read_all_ceilometer_stations(path=ceil_path):
import xarray as xr
from aux_gps import save_ncfile
stations = [x for x in stations_dict.keys()]
da_list = []
for station in stations:
print('reading station {}'.format(station))
da = read_ceilometer_station(path=path, name=station)
da_list.append(da)
ds = xr.merge(da_list)
save_ncfile(ds, path, filename='MLH_from_ceilometers.nc')
return ds
def concat_wrf_vars_same_date(path=des_path, date='2014-08-16'):
import xarray as xr
from aux_gps import path_glob
from aux_gps import save_ncfile
files = path_glob(path, 'wrfout_*_{}_*_*.nc'.format(date))
dsl = [xr.open_dataset(x) for x in files]
ds = xr.merge(dsl)
varnames = '_'.join(sorted([x for x in ds]))
name = files[0].as_posix().split('/')[-1].split('.')[0].split('_')[0:-1]
filename = '_'.join(name) + '_{}'.format(varnames) + '.nc'
save_ncfile(ds, path, filename)
return
def load_wrf_var_from_wrf_file_and_save(file, varname="rh2", savepath=None):
"""load one wrfvar from wrf file and save it to savepath"""
from netCDF4 import Dataset
import wrf
nc = Dataset(file)
from aux_gps import save_ncfile
name = file.as_posix().split('/')[-1].split('.')[0]
filename = '{}_{}.nc'.format(name, varname)
wrfvar = wrf.getvar(wrfin=nc, varname=varname, timeidx=wrf.ALL_TIMES)
if savepath is not None:
if wrfvar.attrs['projection'] is not None:
wrfvar.attrs['projection'] = wrfvar.attrs['projection'].proj4()
save_ncfile(wrfvar, savepath, filename)
return wrfvar
def assemble_WRF_pwv(path=des_path, work_path=work_yuval, radius=1):
from PW_stations import produce_geo_gnss_solved_stations
import xarray as xr
from aux_gps import save_ncfile
from aux_gps import get_nearest_lat_lon_for_xy
from aux_gps import get_unique_index
df = produce_geo_gnss_solved_stations(path=work_path / 'gis', plot=False)
dsea_point = df.loc['dsea'][['lat', 'lon']].astype(float).values
if radius is not None:
point = None
else:
point = dsea_point
wrf_pw = read_all_WRF_GNSS_files(path, var='pw', point=point)
wrf_pw8 = xr.load_dataarray(
path / 'pw_wrfout_d04_2014-08-08_40lev.nc').sel(Time='2014-08-08')
wrf_pw16 = xr.load_dataarray(
path / 'pw_wrfout_d04_2014-08-16_40lev.nc').sel(Time='2014-08-16')
wrf_pw_8_16 = xr.concat([wrf_pw8, wrf_pw16], 'Time')
print('looking for {} at wrf.'.format(dsea_point))
loc = get_nearest_lat_lon_for_xy(wrf_pw_8_16['XLAT'], wrf_pw_8_16['XLONG'],
dsea_point)
print(loc)
if radius is not None:
print('getting {} radius around {}.'.format(radius, dsea_point))
lat_islice = [loc[0][0] - radius, loc[0][0] + radius + 1]
lon_islice = [loc[0][1] - radius, loc[0][1] + radius + 1]
wrf_pw_8_16 = wrf_pw_8_16.isel(south_north=slice(*lat_islice),
west_east=slice(*lon_islice))
loc = get_nearest_lat_lon_for_xy(wrf_pw['XLAT'], wrf_pw['XLONG'],
dsea_point)
lat_islice = [loc[0][0] - radius, loc[0][0] + radius + 1]
lon_islice = [loc[0][1] - radius, loc[0][1] + radius + 1]
wrf_pw = wrf_pw.isel(south_north=slice(*lat_islice),
west_east=slice(*lon_islice))
else:
wrf_pw_8_16 = wrf_pw_8_16.isel(south_north=loc[0][0],
west_east=loc[0][1])
wrf_pw = xr.concat([wrf_pw, wrf_pw_8_16], 'Time')
wrf_pw = wrf_pw.rename({'Time': 'time'})
wrf_pw = wrf_pw.sortby('time')
wrf_pw = get_unique_index(wrf_pw)
if wrf_pw.attrs['projection'] is not None:
wrf_pw.attrs['projection'] = wrf_pw.attrs['projection'].proj4()
if radius is not None:
filename = 'pwv_wrf_dsea_gnss_radius_{}_2014-08.nc'.format(radius)
else:
filename = 'pwv_wrf_dsea_gnss_point_2014-08.nc'
save_ncfile(wrf_pw, des_path, filename)
return wrf_pw
def get_dryz_from_all_stations(gnss_path, savepath):
from aux_gps import save_ncfile
from aux_gps import path_glob
import xarray as xr
pathes = path_glob(gnss_path, '*/')
stations = [x.as_posix().split('/')[-1] for x in pathes]
ds_list = []
for station in stations:
print('obtaining ZHD from station {}'.format(station))
try:
zhd = get_dryz_from_one_station(gnss_path, station=station)
ds_list.append(zhd)
except FileNotFoundError:
continue
ds = xr.merge(ds_list)
filename = 'ZHD_GNSS.nc'
save_ncfile(ds, savepath, filename)
return ds
def produce_pw_all_stations(ds, axis_path, mda_path):
from PW_stations import load_mda
from PW_stations import produce_GNSS_station_PW
from aux_gps import fill_na_xarray_time_series_with_its_group
from aux_gps import path_glob
from aux_gps import save_ncfile
import xarray as xr
# first load mda:
mda = load_mda(mda_path)
# now loop over each station, produce pwv and save:
st_dirs = path_glob(axis_path, '*/')
st_dirs = [x for x in st_dirs if x.is_dir()]
st_dirs = [x for x in st_dirs if not x.as_posix().split('/')[-1].isnumeric()]
assert len(st_dirs) == 27
pwv_list = []
for st_dir in st_dirs:
station = st_dir.as_posix().split('/')[-1]
last_file = sorted(path_glob(st_dir/'dr/ultra', '*.nc'))[-1]
last_file_str = last_file.as_posix().split('/')[-1][4:13]
wet = xr.load_dataset(last_file)['WetZ'].squeeze(drop=True)
logger.info('loaded {}.'.format(last_file))
wet_error = xr.load_dataset(last_file)['WetZ_error'].squeeze(drop=True)
wet.name = station
wet_error.name = station
# resample temp to 5 mins and reindex to wet delay time:
t = ds[station].resample(time='5T').ffill().reindex_like(wet.time)
# fill in NaNs with mean hourly signal:
t_new = fill_na_xarray_time_series_with_its_group(t, grp='hour')
try:
pwv = produce_GNSS_station_PW(wet, t_new, mda=mda,
model_name='LR', plot=False)
pwv_error = produce_GNSS_station_PW(wet_error, t_new, mda=mda,
model_name='LR', plot=False)
pwv_error.name = '{}_error'.format(pwv.name)
pwv_ds = xr.merge([pwv, pwv_error])
filename = '{}{}_PWV.nc'.format(station, last_file_str)
save_ncfile(pwv_ds, st_dir/'dr/ultra', filename)
pwv_list.append(pwv_ds)
except ValueError as e:
logger.warning('encountered error: {}, skipping {}'.format(e, last_file))
continue
dss = xr.merge(pwv_list)
filename = 'AXIS_{}_PWV_ultra.nc'.format(last_file_str)
save_ncfile(dss, axis_path, filename)
def process_ims_stations(mainpath, window, var='TD', ds=None):
import os
import xarray as xr
from aux_gps import path_glob
from axis_process import produce_rinex_filenames_at_time_window
from aux_gps import save_ncfile
import pandas as pd
logger.info('processing IMS stations with {} variable'.format(var))
savepath = mainpath / var
if not savepath.is_dir():
os.mkdir(savepath)
logger.info('created {}.'.format(savepath))
else:
logger.info('{} already exist.'.format(savepath))
if ds is None:
files = path_glob(mainpath, '*.nc')
dsl = [xr.load_dataset(x) for x in files]
else:
dsl = ds
ds_list = []
for ds in dsl:
try:
ds_var = ds[var]
except KeyError:
logger.warning('no {} in {}.'.format(
var, ds.attrs['station_name']))
continue
ds_var.name = ds.attrs['station_name']
ds_var.attrs['lat'] = ds.attrs['lat']
ds_var.attrs['lon'] = ds.attrs['lon']
ds_var.attrs['station_id'] = ds.attrs['station_id']
ds_list.append(ds_var)
ds = xr.merge(ds_list)
now_dt = pd.Timestamp.utcnow().floor('H')
names = produce_rinex_filenames_at_time_window(end_dt=now_dt,
window=window)
st_str = names[0][4:8]
end_str = names[-1][4:8]
filename = 'IMS_{}_{}-{}.nc'.format(var, st_str, end_str)
save_ncfile(ds, savepath, filename)
# finally delete all nc files:
if ds is None:
[x.unlink() for x in files]
return ds
def read_and_concat_smoothFinals(rinexpath, solution='Final'):
import xarray as xr
from aux_gps import save_ncfile
from aux_gps import path_glob
years = [x.as_posix().split('/')[-1] for x in path_glob(rinexpath, '*/')]
years = [x for x in years if x.isnumeric()]
for year in years:
dsl = []
# doys = [x.as_posix().split('/')[-1] for x in path_glob(rinexpath/year, '*/')]
for doypath in path_glob(rinexpath/year, '*/'):
file = doypath / 'dr' / solution / 'smoothFinal.nc'
if file.is_file():
dsl.append(xr.load_dataset(file))
print('found smoothFinal.nc in {}'.format(doypath))
if dsl:
ds = xr.concat(dsl, 'time')
ds = ds.sortby('time')
save_ncfile(ds, rinexpath, 'smoothFinal_{}.nc'.format(year))
return ds
def load_wrf_output_and_save_field(path=des_path, varname="pw", savepath=None):
"""
load WRF output field and save it to savepath
Parameters
----------
path : Path() or str, optional
the WRF loadpath. The default is des_path.
varname : str, optional
can be 'temp', 'pres', etc.. The default is 'pw'.
savepath : Path() or str, optional
The field savepath. The default is None.
Returns
-------
var_list : list
field dataarrays list.
"""
import wrf
import xarray as xr
from aux_gps import path_glob
from aux_gps import save_ncfile
files = path_glob(path, 'wrfout_*.nc')
var_list = []
for file in files:
ds = xr.open_dataset(file)
wrfin = ds._file_obj.ds
wrfvar = wrf.getvar(wrfin=wrfin,
varname=varname,
timeidx=wrf.ALL_TIMES)
if savepath is not None:
if wrfvar.attrs['projection'] is not None:
wrfvar.attrs['projection'] = wrfvar.attrs['projection'].proj4()
filename_to_save = '{}_{}'.format(varname,
file.as_posix().split('/')[-1])
save_ncfile(wrfvar, savepath, filename_to_save)
var_list.append(wrfvar)
return var_list
def produce_and_save_soi_axis_pwv(axis_path=axis_path,
soi_path=dsea_gipsy_path,
ims_path=ims_path,
savepath=work_yuval):
import xarray as xr
from aux_gps import save_ncfile
soi_pwv = produce_pwv_from_dsea_axis_station(path=soi_path,
ims_path=ims_path)
axis_pwv = produce_pwv_from_dsea_axis_station(path=axis_path,
ims_path=ims_path)
soi_pwv.attrs['GNSS network'] = 'SOI-APN'
soi_pwv.attrs['station'] = 'dsea'
soi_pwv.attrs['units'] = 'mm'
soi_pwv = soi_pwv.reset_coords(drop=True)
axis_pwv.attrs['GNSS network'] = 'AXIS'
axis_pwv = axis_pwv.reset_coords(drop=True)
axis_pwv.attrs['station'] = 'dsea'
axis_pwv.attrs['units'] = 'mm'
ds = xr.Dataset()
ds['pwv-soi'] = soi_pwv
ds['pwv-axis'] = axis_pwv
save_ncfile(ds, savepath, 'DSEA_PWV_GNSS_2014-08.nc')
return ds
def agg_month_consecutive_syn_class(path=climate_path, normalize=True):
import numpy as np
import pandas as pd
from aux_gps import save_ncfile
df = read_synoptic_classification(path=path, report=False)
df['month'] = df.index.month
df['year'] = df.index.year
df['months'] = df['year'].astype(str) + '-' + df['month'].astype(str)
new_df = df.groupby(df['months']).apply(find_consecutive_classes)
new_df.columns = ['class_sum']
new_df = new_df.unstack()
new_df.columns = np.arange(1, 20, 1)
dt = pd.to_datetime(new_df.index)
new_df.set_index(dt, inplace=True)
new_df = new_df.sort_index()
new_df.index.name = 'time'
if normalize:
new_df = new_df.divide(new_df.index.days_in_month, axis=0)
da = new_df.to_xarray().to_array('class')
ds = da.to_dataset(name='consecutive')
filename = 'GNSS_synoptic_class_consecutive.nc'
save_ncfile(ds, work_yuval, filename)
return ds
def perform_pwv_filling_last_decade(path=work_yuval,
fyear='2009',
lyear='2019',
drop=['slom', 'elro']):
import xarray as xr
from aux_gps import save_ncfile
pw = xr.load_dataset(path / 'GNSS_PW_monthly_thresh_50.nc')
pw = pw.sel(time=slice(fyear, lyear))
pw = pw.drop_vars(drop)
prepare_pwv_for_climatol(freq='monthly',
first_year=fyear,
last_year=lyear,
pwv_ds=pw)
# then run these two lines in R:
# homogen('PWV',2009,2019, na.strings="-999.9",dz.max=7,std=2)
# dahstat('PWV',2009,2019,stat='series',long=TRUE)
ds, ds_flag = read_climatol_results(first_year=fyear, last_year=lyear)
filename = 'GNSS_PW_monthly_homogenized_filled_{}-{}.nc'.format(
fyear, lyear)
save_ncfile(ds, path, filename)
filename = 'GNSS_PW_monthly_homogenized_filled_flags_{}-{}.nc'.format(
fyear, lyear)
save_ncfile(ds_flag, path, filename)
return
def ims_download(savepath, window, save=False):
"""
Downloads a 10mins parameter from the IMS for the last <window> hours for all stations.
Parameters
----------
savepath : Path or string
a full path to download the files, e.g., /home/ziskin/Work_Files/PW_yuval/IMS_T/10mins.
Returns
-------
None.
"""
import pandas as pd
import requests
from requests.exceptions import SSLError
from requests.exceptions import ConnectionError
from aux_gps import save_ncfile
now_dt = pd.Timestamp.now().floor('H')
start_dt = now_dt - pd.Timedelta('{} hour'.format(window))
logger.info('Downloading IMS to {} and window {}'.format(
savepath, window))
logger.info('Fetching IMS from {} to {}.'.format(start_dt, now_dt))
# use API from IMS
myToken = 'f058958a-d8bd-47cc-95d7-7ecf98610e47'
headers = {'Authorization': 'ApiToken ' + myToken}
# download meta-data on stations:
r = requests.get('https://api.ims.gov.il/v1/envista/stations/',
headers=headers)
stations_df = pd.DataFrame(r.json())
# use only active 10mins stations:
stations_df = stations_df[stations_df['timebase'] == 10]
stations_df = stations_df[stations_df['active']]
ds_list = []
for i, row in stations_df.iterrows():
st_id = row['stationId']
st_name = row['name']
last = now_dt.strftime('%Y/%m/%dT%H:00:00')
first = start_dt.strftime('%Y/%m/%dT%H:00:00')
lat = row['location']['latitude']
if lat is None:
lat = ''
lon = row['location']['longitude']
if lon is None:
lon = ''
dl_command = ('https://api.ims.gov.il/v1/envista/stations/' +
str(st_id) + '/data/?from=' + first + '&to=' + last)
try:
r = requests.get(dl_command, headers=headers)
except SSLError:
logger.warning('SSLError')
r = requests.get(dl_command, headers=headers)
except ConnectionError:
logger.warning('ConnectionError')
r = requests.get(dl_command, headers=headers)
if r.status_code == 204: # i.e., no content:
logger.warning('no content for this search, skipping...')
continue
logger.info('parsing data from {} to dataframe.'.format(st_name))
ds = parse_single_station(r.json()['data'])
ds.attrs['station_name'] = '-'.join(st_name.split(' '))
ds.attrs['lat'] = lat
ds.attrs['lon'] = lon
ds.attrs['station_id'] = st_id
if save:
filename = '{}.nc'.format('-'.join(st_name.split(' ')))
save_ncfile(ds, savepath, filename)
else:
ds_list.append(ds)
return ds_list
