def test_climatology_interpolate_leapday():
ts = pd.Series(np.arange(365),
index=pd.date_range('2001-01-01', freq='D', periods=365))
# remove a part of the time series
clim = anomaly.calc_climatology(ts,
wraparound=True,
respect_leap_years=False,
fill=-1)
assert clim[60] == -1
clim = anomaly.calc_climatology(ts,
wraparound=True,
respect_leap_years=True,
fill=-1)
assert clim[366] == -1
clim = anomaly.calc_climatology(ts,
wraparound=True,
respect_leap_years=False,
fill=-1,
interpolate_leapday=True)
assert clim[60] == np.mean((clim[59], clim[61]))
clim = anomaly.calc_climatology(ts,
wraparound=True,
respect_leap_years=True,
fill=-1,
interpolate_leapday=True)
assert clim[366] == np.mean((clim[365], clim[1]))
def test_climatology_interpolate_leapday():
ts = pd.Series(np.arange(365), index=pd.date_range(
'2001-01-01', freq='D', periods=365))
# remove a part of the time series
clim = anomaly.calc_climatology(ts, wraparound=True, respect_leap_years=False, fill=-1)
assert clim[60] == -1
clim = anomaly.calc_climatology(ts, wraparound=True, respect_leap_years=True, fill=-1)
assert clim[366] == -1
clim = anomaly.calc_climatology(ts, wraparound=True, respect_leap_years=False, fill=-1, interpolate_leapday=True)
assert clim[60] == np.mean((clim[59], clim[61]))
clim = anomaly.calc_climatology(ts, wraparound=True, respect_leap_years=True, fill=-1, interpolate_leapday=True)
assert clim[366] == np.mean((clim[365], clim[1]))
def test_monthly_climatology_always_12():
ts = pd.Series(np.sin(np.arange(366) / 366. * 2 * np.pi),
index=pd.date_range('2000-01-01', freq='D', periods=366))
# remove a part of the time series
ts['2000-02-01':'2000-02-28'] = np.nan
ts = ts.dropna()
clim = anomaly.calc_climatology(ts, unit="month")
assert clim.size == 12
# this should also be the case if interpolate_leapday is set
ts = pd.Series(np.sin(np.arange(10)),
index=pd.date_range('2000-01-01', freq='D', periods=10))
clim = anomaly.calc_climatology(ts, unit="month")
assert clim.size == 12
def calc_anom(df, variable=None):
"""
Calculates anomaly based on climatology for time series.
Parameters
----------
df : pandas DataFrame
Dataframe containing time series.
variable : str
Variable to select from DataFrame
Returns
-------
df : pandas DataFrame
Anomaly of time series.
"""
climatology = calc_climatology(df)
if variable is None:
variable = df.keys()[0]
anom = calc_anomaly(df[variable], climatology=climatology)
df[variable] = anom
columns = []
for cols in df.columns:
columns.append(cols + '_anomaly')
df.columns = columns
return df
def calc_anom(df, variable=None):
"""
Calculates anomaly based on climatology for time series.
Parameters
----------
df : pandas DataFrame
Dataframe containing time series.
variable : str
Variable to select from DataFrame
Returns
-------
df : pandas DataFrame
Anomaly of time series.
"""
climatology = calc_climatology(df)
if variable is None:
variable = df.keys()[0]
anom = calc_anomaly(df[variable], climatology=climatology)
df[variable] = anom
columns = []
for cols in df.columns:
columns.append(cols + '_anomaly')
df.columns = columns
return df
def test_climatology_always_366():
ts = pd.Series(np.sin(np.arange(366) / 366. * 2 * np.pi),
index=pd.date_range('2000-01-01', freq='D', periods=366))
# remove a part of the time series
ts['2000-02-01':'2000-02-28'] = np.nan
ts = ts.dropna()
clim = anomaly.calc_climatology(ts)
assert clim.size == 366
def test_climatology_always_366():
ts = pd.Series(np.sin(np.arange(366) / 366. * 2 * np.pi), index=pd.date_range(
'2000-01-01', freq='D', periods=366))
# remove a part of the time series
ts['2000-02-01': '2000-02-28'] = np.nan
ts = ts.dropna()
clim = anomaly.calc_climatology(ts)
assert clim.size == 366
def calc_anom(self, data):
if self.columns is None:
ite = data
else:
ite = self.columns
for column in ite:
clim = calc_climatology(data[column], **self.kwargs)
data[column] = calc_anomaly(data[column], climatology=clim)
return data
def calc_anom(self, data):
if self.columns is None:
ite = data
else:
ite = self.columns
for column in ite:
clim = calc_climatology(data[column], **self.kwargs)
data[column] = calc_anomaly(data[column], climatology=clim)
return data
def test_climatology_closed():
ts = pd.Series(np.arange(366),
index=pd.date_range('2000-01-01', freq='D', periods=366))
# remove a part of the time series
ts['2000-02-01':'2000-02-28'] = np.nan
ts = ts.dropna()
clim = anomaly.calc_climatology(ts, wraparound=True)
assert clim.size == 366
# test that the arange was closed during the second moving average
assert clim.iloc[365] - 187.90 < 0.01
def test_climatology_closed():
ts = pd.Series(np.arange(366), index=pd.date_range(
'2000-01-01', freq='D', periods=366))
# remove a part of the time series
ts['2000-02-01': '2000-02-28'] = np.nan
ts = ts.dropna()
clim = anomaly.calc_climatology(ts, wraparound=True)
assert clim.size == 366
# test that the arange was closed during the second moving average
assert clim.iloc[365] - 187.90 < 0.01
def calc_anom(Ser,
mode='climatological',
window_size=35,
return_clim=False,
return_clim366=False):
'''
:param Ser: pandas.Series; index must be a datetime index
:param mode: string; one of:
"climatological": calculate anomalies from the mean seasonal cycle
"longterm": inter-annual variabilities only (climatological anomalies minus short-term anomalies)
"shortterm": residuals from the seasonality (i.e., moving average) of each individual year
:param window_size: integer; window size for calculating the climatology and/or seasonality
:param return_clim: boolean; If true, the climatology value is returned for each timestep of the input Series
This overrules the "mode" keyword!
:param return_clim366: boolean; If true, the actual climatology is returned (366 values)
This overrules both the "mode" and "return_clim" keywords!
'''
if mode not in ['climatological', 'longterm', 'shortterm']:
logging.error('calc_anom: unknown anomaly type')
return None
# Calculate the climatology
if (mode != 'shortterm') | return_clim | return_clim366:
clim = calc_climatology(Ser,
respect_leap_years=True,
wraparound=True,
moving_avg_clim=window_size,
fillna=False)
else:
clim = None
# Return the actual climatology (366 values)
if return_clim366:
return clim
# Calculate either climatological or short-term anomalies
res = calc_anom_pytesmo(Ser,
climatology=clim,
window_size=window_size,
return_clim=return_clim)
# Derive long-term anomalies by subtracting short-term anomalies from climatological anomalies
if (mode == 'longterm') and not return_clim:
res -= calc_anom_pytesmo(Ser,
climatology=None,
window_size=window_size)
# Return climatology values for each time step of the input Series
if return_clim:
res = res['climatology']
res.name = Ser.name
return res
def _adapt(self, data):
data = super()._adapt(data)
if self.columns is None:
ite = data
else:
ite = self.columns
for column in ite:
clim = calc_climatology(data[column], **self.kwargs)
data[column] = calc_anomaly(data[column], climatology=clim)
return data
def getEVIanom(lon,lat):
# load datasetes
modis = ee.ImageCollection("MODIS/006/MOD13Q1")
roi = ee.Geometry.Polygon([[[11.51092529296875, 46.626963598343174],
[11.429901123046875, 46.47207397310214],
[11.605682373046875, 46.41529659311316],
[11.734771728515625, 46.442746381846845],
[11.786956787109375, 46.55996197966659]]])
poi = ee.Geometry.Point([lon, lat]).buffer(250)
# mask modis
modis = modis.map(lambda image: image.updateMask(image.select('SummaryQA').lt(2))) \
.filterBounds(roi)
# get time-series
def getSeries(image):
out_feature = image.reduceRegion(ee.Reducer.mean(), poi, 250)
return ee.Feature(None, {'result': out_feature})
# get the EVI time-series
evi_series = modis.map(getSeries).getInfo()
EVI = np.array([x['properties']['result']['EVI'] for x in evi_series['features']], dtype=np.float)
ge_dates = np.array([datetime.strptime(x['id'], '%Y_%m_%d') for x in evi_series['features']])
valid = np.where(np.isfinite(EVI))
# cut out invalid values
EVI = EVI[valid] / 10000
ge_dates = ge_dates[valid]
# insert into pandas series
EVI = pd.Series(EVI, index=ge_dates)
# calculate climatology
EVI_filled = EVI.reindex(index=pd.date_range(start=datetime(year=2000,month=1,day=1),end=datetime(year=2017,month=12,day=31)), copy=True) \
.interpolate()
evi_clim = calc_climatology(EVI_filled, moving_avg_orig=30, moving_avg_clim=0)
evi_clim_dates = [evi_clim.values[x.dayofyear-1] for x in EVI.index]
evi_clim_dates = pd.Series(evi_clim_dates, index=EVI.index)
evi_clim_dates_f = [evi_clim.values[x.dayofyear - 1] for x in EVI_filled.index]
evi_clim_dates_f = pd.Series(evi_clim_dates_f, index=EVI_filled.index)
evi_anomalies = evi_clim_dates - EVI
return (EVI, evi_clim_dates, evi_anomalies, evi_clim_dates_f, evi_clim)
def plot_freq_components(dir_out):
ds = GEOSldas_io('ObsFcstAna',
exp='NLv4_M36_US_DA_SMAP_Pcorr_LTST').timeseries
ts = ds['obs_fcst'][:, 0, 50, 120].to_pandas().dropna()
clim = calc_climatology(ts)
anom_lt = calc_anom(ts, mode='longterm')
anom_st = calc_anom(ts, mode='shortterm')
clim = calc_anom(ts, return_clim=True)
anom_lst = anom_lt + anom_st
anom_lst.name = 'Anomalies (HF + LF)'
f, axes = plt.subplots(figsize=(18, 6), nrows=3, ncols=1, sharex=True)
fontsize = 11
df = pd.concat((ts, clim), axis=1)
df.columns = ['$T_b$ signal', 'Climatology']
df.plot(ax=axes[0],
color=['darkorange', 'blue'],
linewidth=1.7,
fontsize=fontsize).legend(loc='upper right', fontsize=fontsize)
df = pd.DataFrame(anom_lst)
df.plot(ax=axes[1],
color='crimson',
ylim=[-18, 18],
linewidth=1.7,
fontsize=fontsize).legend(loc='upper right', fontsize=fontsize)
axes[1].axhline(color='black', linestyle='--', linewidth=0.8)
df = pd.concat((anom_st, anom_lt), axis=1)
df.columns = ['HF signal', 'LF signal']
df.plot(ax=axes[2],
color=['goldenrod', 'teal'],
ylim=[-15, 15],
linewidth=1.7,
fontsize=fontsize).legend(loc='upper right', fontsize=fontsize)
axes[2].axhline(color='black', linestyle='--', linewidth=0.8)
plt.xlabel('')
plt.minorticks_off()
plt.xlim('2015-04', '2021,04')
plt.xticks(fontsize=fontsize)
f.savefig(dir_out / f'frequency_components.png',
dpi=300,
bbox_inches='tight')
plt.close()
def plot_innov_ts_example():
lat, lon = 41.70991616028507, -92.39133686043398
io_smap = LDAS_io('ObsFcstAna', exp='US_M36_SMOS40_DA_cal_scaled')
# io_smap = LDAS_io('ObsFcstAna', exp='US_M36_SMOS_DA_cal_scaled_yearly')
idx_lon, idx_lat = io_smap.grid.lonlat2colrow(lon, lat, domain=True)
print(idx_lon, idx_lat)
ts = io_smap.timeseries.isel(lat=idx_lat, lon=idx_lon, species=0).to_dataframe()[['obs_fcst', 'obs_obs']].dropna()
plt.figure(figsize=(21, 8))
ax = plt.subplot(2, 1, 1)
sns.lineplot(data=ts, dashes=False, ax=ax)
plt.title(f'{lat:.2f} N, {lon:.2f} W')
plt.xlabel('')
plt.ylabel('Tb')
# ---- cliamtology ----
ts['obs_fcst_clim'] = calc_anomaly(ts['obs_fcst'], return_clim=True, climatology=calc_climatology(ts['obs_fcst']))[
'climatology']
ts['obs_obs_clim'] = calc_anomaly(ts['obs_obs'], return_clim=True, climatology=calc_climatology(ts['obs_obs']))[
'climatology']
ts['obs_fcst_seas'] = ts['obs_fcst'] - calc_anomaly(ts['obs_fcst'])
ts['obs_obs_seas'] = ts['obs_obs'] - calc_anomaly(ts['obs_obs'])
ax = plt.subplot(2, 1, 2)
ts['climatology_scaled'] = ts['obs_obs'] - ts['obs_obs_clim'] + ts['obs_fcst_clim'] - ts['obs_fcst']
ts['seasonality_scaled'] = ts['obs_obs'] - ts['obs_obs_seas'] + ts['obs_fcst_seas'] - ts['obs_fcst']
sns.lineplot(data=ts[['climatology_scaled', 'seasonality_scaled']], dashes=False, ax=ax)
plt.axhline(color='black', linewidth=1, linestyle='--')
plt.xlabel('')
plt.ylabel('O-F')
plt.tight_layout()
plt.show()
def plot_clim_anom(df, clim=None, axes=None, markersize=0.75,
mfc='0.3', mec='0.3', clim_color='0.0',
clim_linewidth=0.5, clim_linestyle='-',
pos_anom_color='#799ADA', neg_anom_color='#FD8086',
anom_linewidth=0.2, add_titles=True):
"""
Takes a pandas DataFrame and calculates the climatology and anomaly
and plots them in a nice way for each column
Parameters
----------
df : pandas.DataFrame
clim : pandas.DataFrame, optional
if given these climatologies will be used
if not given then climatologies will be calculated
this DataFrame must have the same number of columns as df
and also the column names.
each climatology must have doy as index.
axes : list of matplotlib.Axes, optional
list of axes on which each column should be plotted
if not given a standard layout is generated
markersize : float, optional
size of the markers for the datapoints
mfc : matplotlib color, optional
markerfacecolor, color of the marker face
mec : matplotlib color, optional
markeredgecolor
clim_color : matplotlib color, optional
color of the climatology
clim_linewidth : float, optional
linewidth of the climatology
clim_linestyle : string, optional
linestyle of the climatology
pos_anom_color : matplotlib color, optional
color of the positive anomaly
neg_anom_color : matplotlib color, optional
color of the negative anomaly
anom_linewidth : float, optional
linewidth of the anomaly lines
add_titles : boolean, optional
if set each subplot will have it's column name as title
Default : True
Returns
-------
Figure : matplotlib.Figure
if no axes were given
axes : list of matploblib.Axes
if no axes were given
"""
if type(df) == pd.Series:
df = pd.DataFrame(df)
nr_columns = len(df.columns)
# make own axis if necessary
if axes is None:
own_axis = True
gs = gridspec.GridSpec(nr_columns, 1, right=0.8)
fig = plt.figure(num=None, figsize=(6, 2 * nr_columns),
dpi=150, facecolor='w', edgecolor='k')
last_axis = fig.add_subplot(gs[nr_columns - 1])
axes = []
for i, grid in enumerate(gs):
if i < nr_columns - 1:
ax = fig.add_subplot(grid, sharex=last_axis)
axes.append(ax)
ax.xaxis.set_visible(False)
axes.append(last_axis)
else:
own_axis = False
for i, column in enumerate(df):
Ser = df[column]
ax = axes[i]
if clim is None:
clima = anom.calc_climatology(Ser)
else:
clima = pd.Series(clim[column])
anomaly = anom.calc_anomaly(Ser, climatology=clima, return_clim=True)
anomaly[Ser.name] = Ser
anomaly = anomaly.dropna()
pos_anom = anomaly[Ser.name].values > anomaly['climatology'].values
neg_anom = anomaly[Ser.name].values < anomaly['climatology'].values
ax.plot(anomaly.index, anomaly[Ser.name].values, 'o',
markersize=markersize, mfc=mfc, mec=mec)
ax.plot(anomaly.index, anomaly['climatology'].values,
linestyle=clim_linestyle,
color=clim_color,
linewidth=clim_linewidth)
ax.fill_between(anomaly.index,
anomaly[Ser.name].values,
anomaly['climatology'].values, interpolate=True,
where=pos_anom, color=pos_anom_color,
linewidth=anom_linewidth)
ax.fill_between(anomaly.index,
anomaly[Ser.name].values,
anomaly['climatology'].values, interpolate=True,
where=neg_anom, color=neg_anom_color,
linewidth=anom_linewidth)
if add_titles:
ax.set_title(column)
if own_axis:
return fig, axes
else:
return None, None
def compare_data(ismn_data, validation_data, scaling='linreg', anomaly=None):
"""
Compare data from an ISMN station to the defined validation datasets.
Parameters
----------
ismn_data: pandas.Dataframe
Data from the ISMN used as a reference
validation_data: dict
Dictionary of pandas.DataFrames, One for each dataset to
compare against
scaling: string, optional
Scaling method to use.
anomaly: string
If set then the validation is done for anomalies.
"""
insitu_label = 'soil moisture'
if anomaly != None:
if anomaly == 'climatology':
ascat_clim = anomaly_calc.calc_climatology(
ascat_masked[ascat_label])
insitu_clim = anomaly_calc.calc_climatology(
ismn_data['soil moisture'])
ascat_anom = anomaly_calc.calc_anomaly(ascat_masked[ascat_label],
climatology=ascat_clim)
ascat_masked[ascat_label] = ascat_anom.values
insitu_anom = anomaly_calc.calc_anomaly(ISMN_data['insitu'],
climatology=insitu_clim)
ISMN_data['insitu'] = insitu_anom.values
if anomaly == 'average':
ascat_anom = anomaly_calc.calc_anomaly(ascat_masked[ascat_label])
ascat_masked[ascat_label] = ascat_anom.values
insitu_anom = anomaly_calc.calc_anomaly(ISMN_data['insitu'])
ISMN_data['insitu'] = insitu_anom.values
ascat_masked = ascat_masked.dropna()
ISMN_data = ISMN_data.dropna()
for dname in validation_data:
vdata = validation_data[dname]
vdata_label = 'cci_sm'
matched_data = temp_match.matching(ismn_data, vdata, window=1)
if scaling != 'noscale' and scaling != 'porosity':
scaled_data = scale.add_scaled(matched_data,
label_in=vdata_label,
label_scale=insitu_label,
method=scaling)
scaled_label = vdata_label + '_scaled_' + scaling
scaled_data = scaled_data[[insitu_label, scaled_label]]
elif scaling == 'noscale':
scaled_data = matched_data[[insitu_label, vdata_label]]
scaled_label = vdata_label
# scaled_data.rename(columns={'insitu': ISMN_ts_name}, inplace=True)
labels, values = scaled_data.to_dygraph_format()
ascat_insitu = {'labels': labels, 'data': values}
x, y = scaled_data[insitu_label].values, scaled_data[scaled_label].values
kendall, p_kendall = sc_stats.kendalltau(x.tolist(), y.tolist())
spearman, p_spearman = sc_stats.spearmanr(x, y)
pearson, p_pearson = sc_stats.pearsonr(x, y)
rmsd = metrics.rmsd(x, y)
bias = metrics.bias(y, x)
mse, mse_corr, mse_bias, mse_var = metrics.mse(x, y)
statistics = {
'kendall': {
'v': '%.2f' % kendall,
'p': '%.4f' % p_kendall
},
'spearman': {
'v': '%.2f' % spearman,
'p': '%.4f' % p_spearman
},
'pearson': {
'v': '%.2f' % pearson,
'p': '%.4f' % p_pearson
},
'bias': '%.4f' % bias,
'rmsd': {
'rmsd': '%.4f' % np.sqrt(mse),
'rmsd_corr': '%.4f' % np.sqrt(mse_corr),
'rmsd_bias': '%.4f' % np.sqrt(mse_bias),
'rmsd_var': '%.4f' % np.sqrt(mse_var)
},
'mse': {
'mse': '%.4f' % mse,
'mse_corr': '%.4f' % mse_corr,
'mse_bias': '%.4f' % mse_bias,
'mse_var': '%.4f' % mse_var
}
}
scaling_options = {
'noscale': 'No scaling',
'porosity': 'Scale using porosity',
'linreg': 'Linear Regression',
'mean_std': 'Mean - standard deviation',
'min_max': 'Minimum,maximum',
'lin_cdf_match': 'Piecewise <br> linear CDF matching',
'cdf_match': 'CDF matching'
}
settings = {
'scaling': scaling_options[scaling],
# 'snow_depth': mask['snow_depth'],
# 'surface_temp': mask['st_l1'],
# 'air_temp': mask['air_temp']
}
era_data = {'labels': [], 'data': []}
output_data = {
'validation_data': ascat_insitu,
'masking_data': era_data,
'statistics': statistics,
'settings': settings
}
return output_data, 1
marker='o',
color='#00bfff',
label='SSM')
ts_ascat_sm['swi_t10'].plot(ax=ax, lw=2, label='SWI T=10')
ts_ascat_sm['swi_t50'].plot(ax=ax, lw=2, label='SWI T=50')
plt.legend()
# Calculate anomaly based on moving +- 17 day window
anomaly_ascat = anomaly.calc_anomaly(ts_ascat['sm'], window_size=35)
fig2, ax = plt.subplots(1, 1, figsize=(15, 5))
anomaly_ascat.plot(ax=ax, lw=2, label='ASCAT SM Anomaly')
plt.legend()
# Calculate climatology
ts_ascat_clim = ts_ascat.dropna()
climatology_ascat = anomaly.calc_climatology(ts_ascat_clim['sm'])
fig3, ax = plt.subplots(1, 1, figsize=(15, 5))
climatology_ascat.plot(ax=ax, lw=2, label='ASCAT SM Climatology')
plt.legend()
# Calculate anomaly based on climatology
ts_ascat_clim = ts_ascat.dropna()
anomaly_clim_ascat = anomaly.calc_anomaly(ts_ascat_clim['sm'],
climatology=climatology_ascat)
fig4, ax = plt.subplots(1, 1, figsize=(15, 5))
anomaly_clim_ascat.plot(ax=ax,
lw=2,
label='ASCAT SM Anomaly vs Climatology')
plt.legend()
print('ciao')
def ismn_validation_run(bsize=500,
name='500m',
fvect1=None,
fvect2=None,
fvect1desc=None,
fvect2desc=None):
s1path = 117
#bsize=250
basepath = '//projectdata.eurac.edu/projects/ESA_TIGER/S1_SMC_DEV/Processing/S1ALPS/ISMN/S1AB_' + name + '_reprocess_lt_05/'
# outpath
outpath = basepath + 'w_GLDAS_asc_desc/'
# Calculate prediction standar deviations
calcstd = False
# use descending orbits
desc = False
# calculate anomalies?
calc_anomalies = False
# initialise available ISMN data
ismn = ismn_interface.ISMN_Interface(
'T:/ECOPOTENTIAL/reference_data/ISMN/')
# get list of networks
networks = ismn.list_networks()
# initialise S1 SM retrieval
# mlmodel = pickle.load(open('/mnt/SAT/Workspaces/GrF/Processing/S1ALPS/ASCAT/gee/mlmodel0.p', 'rb'))
mlmodel_avg = "//projectdata.eurac.edu/projects/ESA_TIGER/S1_SMC_DEV/Processing/S1ALPS/ISMN/S1AB_1km_reprocess_lt_05/no_GLDAS_RFmlmodelNoneSVR_2step.p"
mlmodel = basepath + "w_GLDAS_RFmlmodelNoneSVR_2step.p"
mlmodel_desc = "//projectdata.eurac.edu/projects/ESA_TIGER/S1_SMC_DEV/Processing/S1ALPS/ISMN/S1AB_50m_reprocess_lt_05_descending/w_GLDAS_RFmlmodelNoneSVR_2step.p"
# initialse text report
txtrep = open(outpath + '2_report.txt', 'w')
txtrep.write(
'Accuracy report for Soil Moisture validation based on ISMN stations\n\n'
)
txtrep.write('Model used: ' + mlmodel + '\n')
txtrep.write(
'------------------------------------------------------------------------\n\n'
)
txtrep.write('Name, R, RMSE\n')
xyplot = pd.DataFrame()
cntr = 1
#used_stations = np.load('X:/Workspaces/GrF/Processing/S1ALPS/ISMN/gee_global_all_no_deep/ValidStaions.npy')
used_stations = pickle.load(open(basepath + "testset_meta.p", 'rb'))
#invalid_col = pickle.load(open('/mnt/SAT/Workspaces/GrF/Processing/S1ALPS/ISMN/gee_global005_highdbtollerance/invalid_col.p', 'rb'))
#used_stations = [invalid_col['ntwkname'][i] + ', ' + invalid_col['stname'][i] for i in range(len(invalid_col['ntwkname']))]
#for ntwk in networks:
#for vstation in used_stations:
s1_ts_list = list()
station_ts_list = list()
station_name_list = list()
gldas_ts_list = list()
s1_failed_list = list()
s1_ub_list = list()
for st_i in range(len(used_stations[0])):
#for st_i in [0]:
#ntwk, st_name = [x.strip() for x in vstation.split(',')]
ntwk = used_stations[1][st_i]
st_name = used_stations[0][st_i]
# if st_name != 'ROSASCYN':# and st_name != 'Salem-10-W':
# continue
# get list of available stations
available_stations = ismn.list_stations(ntwk)
# iterate through all available ISMN stations
#for st_name in available_stations:
try:
station = ismn.get_station(st_name, ntwk)
station_vars = station.get_variables()
# if st_name != '2.10':
# continue
# if (st_name != 'Boulder-14-W'):
# continue
#if st_name not in ['ANTIMONYFL', 'CALIVALLEY', 'Bethlehem']:
# if st_name not in ['CALIVALLEY']:
# continue
if 'soil moisture' not in station_vars:
continue
station_depths = station.get_depths('soil moisture')
# if 0.0 in station_depths[0]:
# sm_sensors = station.get_sensors('soil moisture', depth_from=0, depth_to=0.24)
# station_ts = station.read_variable('soil moisture', depth_from=0, depth_to=0.24, sensor=sm_sensors[0])
# else:
# continue
if 0.0 in station_depths[0]:
did = np.where(station_depths[0] == 0.0)
dto = station_depths[1][did]
sm_sensors = station.get_sensors('soil moisture',
depth_from=0,
depth_to=dto[0])
print(sm_sensors[0])
station_ts = station.read_variable('soil moisture',
depth_from=0,
depth_to=dto[0],
sensor=sm_sensors[0])
elif 0.05 in station_depths[0]:
sm_sensors = station.get_sensors('soil moisture',
depth_from=0.05,
depth_to=0.05)
station_ts = station.read_variable('soil moisture',
depth_from=0.05,
depth_to=0.05,
sensor=sm_sensors[0])
else:
continue
print(st_name)
plotpath = outpath + st_name + '.png'
# if os.path.exists(plotpath):
# continue
# get station ts
station_ts = station_ts.data['soil moisture']
# get S1 time series
s1_ts, s1_ts_std, outliers, s1_failed = extract_time_series_gee(
mlmodel,
mlmodel_avg,
'/mnt/SAT4/DATA/S1_EODC/',
outpath,
station.latitude,
station.longitude,
name=st_name,
footprint=bsize,
calcstd=calcstd,
desc=desc,
target=station_ts,
feature_vect1=fvect1,
feature_vect2=fvect2) #,
#s1path=s1path)
s1_ts2, s1_ts_std2, outliers2, s1_failed2 = extract_time_series_gee(
mlmodel_desc,
mlmodel_avg,
'/mnt/SAT4/DATA/S1_EODC/',
outpath,
station.latitude,
station.longitude,
name=st_name,
footprint=bsize,
calcstd=calcstd,
desc=True,
target=station_ts,
feature_vect1=fvect1desc,
feature_vect2=fvect2desc) # ,
if (s1_ts is not None) and (s1_ts2 is not None):
# correct the mean offset between time-series from ascending and descending orbits
meandiff = s1_ts.mean() - s1_ts2.mean()
s1_ts2 = s1_ts2 + meandiff
meandiff_failed = s1_failed.median() - s1_failed2.median()
s1_failed2 = s1_failed2 + meandiff_failed
s1_ts = pd.concat([s1_ts, s1_ts2])
s1_failed = pd.concat([s1_failed, s1_failed2])
elif (s1_ts is None) and (s1_ts2 is not None):
s1_ts = s1_ts2
s1_failed = s1_failed2
s1_ts.sort_index(inplace=True)
s1_failed.sort_index(inplace=True)
if s1_ts is None:
continue
if len(s1_ts) < 5:
continue
#evi_ts = extr_MODIS_MOD13Q1_ts_GEE(station.longitude, station.latitude, bufferSize=150)
#evi_ts = pd.Series(evi_ts[1]['EVI'], index=evi_ts[0])
gldas_ts = extr_GLDAS_ts_GEE(
station.longitude,
station.latitude,
bufferSize=150,
yearlist=[2014, 2015, 2016, 2017, 2018, 2019])
gldas_ts = gldas_ts / 100.
start = np.array([s1_ts.index[0], station_ts.index[0]]).max()
end = np.array([s1_ts.index[-1], station_ts.index[-1]]).min()
if start > end:
continue
station_ts = station_ts[start:end]
s1_ts = s1_ts[start:end]
s1_failed = s1_failed[start:end]
#outliers = outliers[start:end]
#evi_ts = evi_ts[start:end]
gldas_ts = gldas_ts[start:end]
if calcstd == True:
s1_ts_std = s1_ts_std[start:end]
if len(s1_ts) < 1:
continue
#station_ts = station_ts.iloc[np.where(station_ts > 0.1)]
#s1_ts = s1_ts[np.where(error_ts == error_ts.min())[0]]
s1_ts_res = s1_ts.resample('D').mean().rename('s1')
station_ts_res = station_ts.resample('D').mean().rename('ismn')
gldas_ts_res = gldas_ts.resample('D').mean().rename('gldas')
if calc_anomalies == True:
from pytesmo.time_series import anomaly as pyan
s1_clim = pyan.calc_climatology(s1_ts_res.interpolate())
station_clim = pyan.calc_climatology(
station_ts_res.interpolate())
s1_ts = pyan.calc_anomaly(s1_ts, climatology=s1_clim)
s1_ts_res = pyan.calc_anomaly(s1_ts_res, climatology=s1_clim)
station_ts = pyan.calc_anomaly(station_ts,
climatology=station_clim)
station_ts_res = pyan.calc_anomaly(station_ts_res,
climatology=station_clim)
# calculate error metrics
ts_bias = s1_ts_res.subtract(station_ts_res).mean()
# cdf matching
tobemerged = [
s1_ts_res.dropna(),
gldas_ts_res.dropna(),
station_ts_res.dropna()
]
s1_and_station = pd.concat(tobemerged, axis=1, join='inner')
statmask = (s1_and_station['ismn'] > 0) & (
s1_and_station['ismn'] < 1) & (s1_and_station['s1'] >
0) & (s1_and_station['s1'] < 1)
p2 = s1_and_station['ismn'][statmask].std(
) / s1_and_station['s1'][statmask].std()
p1 = s1_and_station['ismn'][statmask].mean() - (
p2 * s1_and_station['s1'][statmask].mean())
s1_ts_ub = p1 + (p2 * s1_ts)
s1_and_station['s1ub'] = p1 + (p2 * s1_and_station['s1'])
ts_bias = s1_and_station['s1ub'].subtract(
s1_and_station['ismn']).median()
s1_failed = p1 + (p2 * s1_failed)
xytmp = pd.concat(
{
'y': s1_and_station['s1ub'],
'x': s1_and_station['ismn']
},
join='inner',
axis=1)
if cntr == 1:
xyplot = xytmp
else:
xyplot = pd.concat([xyplot, xytmp], axis=0)
cntr = cntr + 1
ts_cor = s1_and_station['s1ub'].corr(s1_and_station['ismn'])
ts_rmse = np.sqrt(
np.nanmean(
np.square(s1_and_station['s1ub'].subtract(
s1_and_station['ismn']))))
#ts_ubrmse = np.sqrt(np.sum(np.square(s1_ts_res.subtract(s1_ts_res.mean()).subtract(station_ts_res.subtract(station_ts_res.mean())))))
ts_ubrmse = np.sqrt(
np.sum(
np.square((s1_and_station['s1ub'] -
s1_and_station['s1ub'].mean()) -
(s1_and_station['ismn'] -
s1_and_station['ismn'].mean()))) /
len(s1_and_station['s1ub']))
print('R: ' + str(ts_cor))
print('RMSE: ' + str(ts_rmse))
print('Bias: ' + str(ts_bias))
txtrep.write(st_name + ', ' + str(ts_cor) + ', ' + str(ts_rmse) +
'\n')
s1_ts_list.append(s1_ts)
station_ts_list.append(station_ts)
station_name_list.append(st_name)
gldas_ts_list.append(gldas_ts)
s1_failed_list.append(s1_failed)
s1_ub_list.append(s1_ts_ub)
# plot
# plt.figure(figsize=(18, 6))
fig, ax1 = plt.subplots(figsize=(7.16, 1.4), dpi=300)
line1, = ax1.plot(s1_ts_ub.index,
s1_ts_ub,
color='b',
linestyle='',
marker='+',
label='Sentinel-1',
linewidth=0.2)
line8, = ax1.plot(s1_failed.index,
s1_failed,
color='r',
linestyle='',
marker='+',
label='fail',
linewidth=0.2)
line2, = ax1.plot(station_ts.index,
station_ts,
label='In-Situ',
linewidth=0.4)
if np.any(outliers) and outliers is not None:
line6, = ax1.plot(s1_ts.index[outliers],
s1_ts.iloc[outliers],
color='r',
linestyle='',
marker='o')
#line3, = ax1.plot(outliers.index, outliers, color='r', linestyle='', marker='*', label='Outliers')
if calcstd == True:
line4, = ax1.plot(s1_ts.index,
s1_ts - np.sqrt(s1_ts_std),
color='k',
linestyle='--',
linewidth=0.2)
line5, = ax1.plot(s1_ts.index,
s1_ts + np.sqrt(s1_ts_std),
color='k',
linestyle='--',
linewidth=0.2)
#ax3 = ax1.twinx()
line6, = ax1.plot(gldas_ts.index,
gldas_ts,
color='g',
linestyle='--',
label='GLDAS',
linewidth=0.2)
#line3, = ax3.plot(evi_ts.index, evi_ts, linewidth=0.4, color='r', linestyle='--', label='MOD13Q1:EVI')
# ax3.axes.tick_params(axis='y', direction='in', labelcolor='r', right='off', labelright='off')
ax1.set_ylabel('Soil Moisture [m3m-3]', size=8)
smc_max = np.max([s1_ts.max(), station_ts.max()])
if smc_max <= 0.5:
smc_max = 0.5
ax1.set_ylim((0, smc_max))
ax1.text(
0.85,
0.4,
'R=' + '{:03.2f}'.format(ts_cor) +
#'\nRMSE=' + '{:03.2f}'.format(ts_rmse) +
'\nBias=' + '{:03.2f}'.format(ts_bias) + '\nRMSE=' +
'{:03.2f}'.format(ts_rmse),
transform=ax1.transAxes,
fontsize=8)
# ax2.set_ylabel('In-Situ [m3/m3]')
#ax3.set_ylabel('EVI')
#fig.tight_layout()
#plt.legend(handles=[line1, line2], loc='upper left', fontsize=8)#, line3, line6])
plt.title(st_name, fontsize=8)
#plt.show()
plt.tight_layout()
plt.savefig(plotpath, dpi=300)
plt.close()
except:
print('No data for: ' + st_name)
pickle.dump(
(s1_ts_list, s1_ub_list, s1_failed, station_ts_list, gldas_ts_list,
station_name_list),
open(
'C:/Users/FGreifeneder/OneDrive - Scientific Network South Tyrol/1_THESIS/pub3/images_submission2/w_GLDAS_validation_tss_'
+ name + '.p', 'wb'))
scatter_valid = np.where(((xyplot['x'] > 0) & (xyplot['x'] < 1))
& ((xyplot['y'] > 0) & (xyplot['y'] < 1)))
xyplot = xyplot.iloc[scatter_valid]
urmse_scatter = np.sqrt(
np.sum(
np.square((xyplot['y'] - xyplot['y'].mean()) -
(xyplot['x'] - xyplot['x'].mean()))) / len(xyplot['y']))
rmse_scatter = np.sqrt(
np.nanmean(np.square(xyplot['x'].subtract(xyplot['y']))))
r_scatter = xyplot['x'].corr(xyplot['y'])
#plt.figure(figsize=(3.5, 3), dpi=600)
xyplot.plot.scatter(x='x',
y='y',
color='k',
xlim=(0, 1),
ylim=(0, 1),
figsize=(3.5, 3),
s=1,
marker='*')
plt.xlim(0, 1)
plt.ylim(0, 1)
plt.xlabel("$SMC_{Tot}$ [m$^3$m$^{-3}$]", size=8)
plt.ylabel("$SMC^*_{Tot}$ [m$^3$m$^{-3}$]", size=8)
plt.plot([0, 1], [0, 1], 'k--', linewidth=0.8)
plt.text(0.1,
0.5,
'R=' + '{:03.2f}'.format(r_scatter) + '\nRMSE=' +
'{:03.2f}'.format(rmse_scatter),
fontsize=8) # +
#'\nRMSE=' + '{:03.2f}'.format(rmse_scatter), fontsize=8)
plt.tick_params(labelsize=8)
plt.title('True vs. estimated SMC', size=8)
plt.axes().set_aspect('equal', 'box')
plt.tight_layout()
plt.savefig(outpath + '1_scatterplot.png', dpi=600)
plt.close()
txtrep.write(
'------------------------------------------------------------------------\n\n'
)
txtrep.write('Overall performance:\n')
txtrep.write('R = ' + str(xyplot['x'].corr(xyplot['y'])) + '\n')
txtrep.write(
'RMSE = ' +
str(np.sqrt(np.nanmean(np.square(xyplot['x'].subtract(xyplot['y']))))))
txtrep.write('ubRMSE = ' + str(
np.sqrt(
np.sum(
np.square((xyplot['y'] - xyplot['y'].mean()) -
(xyplot['x'] - xyplot['x'].mean()))) /
len(xyplot['y']))))
txtrep.close()
sm_time = cci['time']
#sm_time = np.array([datetime.datetime.fromtimestamp(i*24*60*60) for i in sm_time])
timediff = datetime.datetime(
year=1979, month=1, day=1, hour=0, minute=0,
second=0) - datetime.datetime.fromtimestamp(0)
sm_time = np.array([
datetime.datetime.fromtimestamp(i * 24 * 60 * 60) + timediff
for i in sm_time
])
sm = np.nanmean(sm, axis=(1, 2))
sm_pd = pd.Series(sm, index=sm_time)
sm_pd = sm_pd.dropna()
sm_df = pd.DataFrame({'Soil Moisture': sm_pd.values}, index=sm_pd.index)
sm_climatology = calc_climatology(sm_pd)
sm_climatology = pd.DataFrame({'Soil Moisture': sm_climatology.values},
index=sm_climatology.index)
fig, axs = plt.subplots(figsize=(25, 10))
plot_clim_anom(sm_df,
clim=sm_climatology,
axes=[axs],
clim_color='r',
clim_linewidth='0.9')
#sm_anomalies_monthly.plot(ax=axs, linestyle='-', linewidth=0.5)
plt.savefig('/mnt/SAT/Workspaces/GrF/model_kassel_sm_1980_2015.png')
plt.close()
cci.close()
def plot_clim_anom(df,
clim=None,
axes=None,
markersize=0.75,
mfc='0.3',
mec='0.3',
clim_color='0.0',
clim_linewidth=0.5,
clim_linestyle='-',
pos_anom_color='#799ADA',
neg_anom_color='#FD8086',
anom_linewidth=0.2,
add_titles=True):
"""
Takes a pandas DataFrame and calculates the climatology and anomaly
and plots them in a nice way for each column
Parameters
----------
df : pandas.DataFrame
clim : pandas.DataFrame, optional
if given these climatologies will be used
if not given then climatologies will be calculated
this DataFrame must have the same number of columns as df
and also the column names.
each climatology must have doy as index.
axes : list of matplotlib.Axes, optional
list of axes on which each column should be plotted
if not given a standard layout is generated
markersize : float, optional
size of the markers for the datapoints
mfc : matplotlib color, optional
markerfacecolor, color of the marker face
mec : matplotlib color, optional
markeredgecolor
clim_color : matplotlib color, optional
color of the climatology
clim_linewidth : float, optional
linewidth of the climatology
clim_linestyle : string, optional
linestyle of the climatology
pos_anom_color : matplotlib color, optional
color of the positive anomaly
neg_anom_color : matplotlib color, optional
color of the negative anomaly
anom_linewidth : float, optional
linewidth of the anomaly lines
add_titles : boolean, optional
if set each subplot will have it's column name as title
Default : True
Returns
-------
Figure : matplotlib.Figure
if no axes were given
axes : list of matploblib.Axes
if no axes were given
"""
if type(df) == pd.Series:
df = pd.DataFrame(df)
nr_columns = len(df.columns)
# make own axis if necessary
if axes is None:
own_axis = True
gs = gridspec.GridSpec(nr_columns, 1, right=0.8)
fig = plt.figure(num=None,
figsize=(6, 2 * nr_columns),
dpi=150,
facecolor='w',
edgecolor='k')
last_axis = fig.add_subplot(gs[nr_columns - 1])
axes = []
for i, grid in enumerate(gs):
if i < nr_columns - 1:
ax = fig.add_subplot(grid, sharex=last_axis)
axes.append(ax)
ax.xaxis.set_visible(False)
axes.append(last_axis)
else:
own_axis = False
for i, column in enumerate(df):
Ser = df[column]
ax = axes[i]
if clim is None:
clima = anom.calc_climatology(Ser)
else:
clima = pd.Series(clim[column])
anomaly = anom.calc_anomaly(Ser, climatology=clima, return_clim=True)
anomaly[Ser.name] = Ser
anomaly = anomaly.dropna()
pos_anom = anomaly[Ser.name].values > anomaly['climatology'].values
neg_anom = anomaly[Ser.name].values < anomaly['climatology'].values
ax.plot(anomaly.index,
anomaly[Ser.name].values,
'o',
markersize=markersize,
mfc=mfc,
mec=mec)
ax.plot(anomaly.index,
anomaly['climatology'].values,
linestyle=clim_linestyle,
color=clim_color,
linewidth=clim_linewidth)
ax.fill_between(anomaly.index,
anomaly[Ser.name].values,
anomaly['climatology'].values,
interpolate=True,
where=pos_anom,
color=pos_anom_color,
linewidth=anom_linewidth)
ax.fill_between(anomaly.index,
anomaly[Ser.name].values,
anomaly['climatology'].values,
interpolate=True,
where=neg_anom,
color=neg_anom_color,
linewidth=anom_linewidth)
if add_titles:
ax.set_title(column)
if own_axis:
return fig, axes
else:
return None, None
import matplotlib.pyplot as plt
import os
if __name__ == '__main__':
cur_path = os.path.abspath(os.path.curdir)
input_dataidx_file = os.path.join(cur_path, 'data/ssmi/0673')
gpi = 720360
dat_obj = datasets.DatasetTs(input_dataidx_file)
print dat_obj.dat_data.dtype
print type(dat_obj.dat_data), dat_obj.dat_data.shape
gpi_data = dat_obj.read_ts(gpi)
print gpi_data.dtype, gpi_data.shape
Ser = pd.Series(gpi_data['sm'], index=gpi_data['jd'])
# Adjust the parameters
climatology = anomaly.calc_climatology(Ser)
print climatology
climatology.plot()
plt.show()
anom = anomaly.calc_anomaly(Ser, climatology=climatology)
anom.plot()
plt.show()
print anom
#bcksct_min = -14.8
#bcksct_max = -8.12
bcksct_rng = bcksct_max - bcksct_min
s1pd_scaled = (s1pd - bcksct_min) / bcksct_rng
s1pd_scaled = (s1pd_scaled * 35.) + 5.
# s1pd_scaled = (s1pd - bcksct_min) / bcksct_rng
# s1pd_scaled = (s1pd_scaled * 15.) + 3.
s1pd_scaled = s1pd_scaled.resample('D').mean()
s1pd_scaled.interpolate(inplace=True)
# anomalies
#bcksct_mean = s1pd_scaled.mean()
#s1pd_anom = (s1pd_scaled / bcksct_mean) - 1
climatology = anomaly.calc_climatology(s1pd_scaled,
moving_avg_orig=5,
moving_avg_clim=30)
s1pd_anom = anomaly.calc_anomaly(s1pd_scaled, climatology=climatology)
plt.figure(figsize=(6.3, 3.7))
s1pd_scaled.plot()
plt.xlabel('Date', fontsize=10)
plt.ylabel('SMC [m3/m-3]')
plt.ylim((0, 50))
plt.show()
plt.savefig('/mnt/SAT/Workspaces/GrF/02_Documents/ISRSE2017/s1ts.png', dpi=300)
plt.close()
plt.figure(figsize=(6.3, 3.7))
s1pd_anom.plot()
timex = pd.date_range('30/9/2014', '30/4/2017', freq='D')
if start > end:
continue
station_ts = station_ts[start:end]
s1_ts = s1_ts[start:end]
if calcstd == True:
s1_ts_std = s1_ts_std[start:end]
if len(s1_ts) < 1:
continue
s1_ts_res = s1_ts.resample('D').mean()
station_ts_res = station_ts.resample('D').mean()
if calc_anomalies == True:
from pytesmo.time_series import anomaly as pyan
s1_clim = pyan.calc_climatology(s1_ts_res.interpolate())
station_clim = pyan.calc_climatology(
station_ts_res.interpolate())
s1_ts = pyan.calc_anomaly(s1_ts, climatology=s1_clim)
s1_ts_res = pyan.calc_anomaly(s1_ts_res, climatology=s1_clim)
station_ts = pyan.calc_anomaly(station_ts,
climatology=station_clim)
station_ts_res = pyan.calc_anomaly(station_ts_res,
climatology=station_clim)
# calculate error metrics
ts_bias = s1_ts_res.subtract(station_ts_res).mean()
tobemerged = [s1_ts_res.dropna(), station_ts_res.dropna()]
s1_and_station = pd.concat(tobemerged, axis=1, join='inner')
# <codecell> # Read data for location in northern Italy ascat_ts = ascat_SSM_reader.read(11, 45) #plot soil moisture ascat_ts.data['sm'].plot(title='SSM data') plt.show() # <codecell> #calculate anomaly based on moving +- 17 day window anom = ts_anomaly.calc_anomaly(ascat_ts.data['sm'], window_size=35) anom.plot(title='Anomaly (35-day window)') plt.show() # <codecell> #calculate climatology climatology = ts_anomaly.calc_climatology(ascat_ts.data['sm']) climatology.plot(title='Climatology') plt.show() # <codecell> #calculate anomaly based on climatology anomaly_clim = ts_anomaly.calc_anomaly(ascat_ts.data['sm'], climatology=climatology) anomaly_clim.plot(title='Anomaly (climatology)') plt.show() # <codecell>
def mazia_vaildation_run(bsize=500, name='500m', fvect1=None, fvect2=None):
# outpath
outpath = '//projectdata.eurac.edu/projects/ESA_TIGER/S1_SMC_DEV/Processing/S1ALPS/ISMN/S1AB_' + name + '_reprocess_lt_05/w_GLDAS_station_validation/'
# Calculate prediction standar deviations
calcstd = False
# use descending orbits
desc = False
calc_anomalies = False
# initialise S1 SM retrieval
# mlmodel = pickle.load(open('/mnt/SAT/Workspaces/GrF/Processing/S1ALPS/ASCAT/gee/mlmodel0.p', 'rb'))
mlmodel = "//projectdata.eurac.edu/projects/ESA_TIGER/S1_SMC_DEV/Processing/S1ALPS/ISMN/S1AB_" + name + "_reprocess_lt_05/w_GLDAS_RFmlmodelNoneSVR_2step.p"
# initialse text report
txtrep = open(outpath + '2_Mazia_report.txt', 'w')
txtrep.write(
'Accuracy report for Soil Moisture validation based on ISMN stations\n\n'
)
txtrep.write('Model used: ' + mlmodel + '\n')
txtrep.write(
'------------------------------------------------------------------------\n\n'
)
txtrep.write('Name, R, RMSE\n')
xyplot = pd.DataFrame()
cntr = 1
# define mazia station locations
m_stations = {
'I1': [10.57978, 46.68706],
'I3': [10.58359, 46.68197],
'P1': [10.58295, 46.68586],
'P2': [10.58525, 46.68433],
'P3': [10.58562, 46.68511]
}
m_station_paths = '//projectdata.eurac.edu/projects/ESA_TIGER/S1_SMC_DEV/01_Data/InSitu/MaziaValley_SWC_2015_16/'
s1_ts_list = list()
station_ts_list = list()
station_name_list = list()
gldas_ts_list = list()
for vstation in m_stations:
st_name = vstation
st_coordinates = m_stations[vstation]
try:
# get in situ data
full_path2015 = m_station_paths + st_name + '_YEAR_2015.csv'
insitu2015 = pd.read_csv(full_path2015,
header=0,
skiprows=[0, 2, 3],
index_col=0,
parse_dates=True,
sep=',')
full_path2016 = m_station_paths + st_name + '_YEAR_2016.csv'
insitu2016 = pd.read_csv(full_path2016,
header=0,
skiprows=[0, 2, 3],
index_col=0,
parse_dates=True,
sep=',')
insitu = insitu2015.append(insitu2016)
# get station ts
station_ts = pd.Series(
insitu[['SWC_02_A_Avg', 'SWC_02_B_Avg',
'SWC_02_C_Avg']].mean(axis=1))
plotpath = outpath + st_name + '.png'
s1_ts, s1_ts_std, outliers = extract_time_series_gee(
mlmodel,
mlmodel,
'/mnt/SAT4/DATA/S1_EODC/',
outpath,
st_coordinates[1],
st_coordinates[0],
name=st_name,
footprint=bsize,
calcstd=calcstd,
desc=desc,
target=station_ts,
feature_vect1=fvect1,
feature_vect2=fvect2) # ,
if s1_ts is None:
continue
if len(s1_ts) < 5:
continue
gldas_ts = extr_GLDAS_ts_GEE(st_coordinates[1],
st_coordinates[0],
bufferSize=150,
yearlist=[2015, 2016])
gldas_ts = gldas_ts / 100.
start = np.array([s1_ts.index[0], station_ts.index[0]]).max()
end = np.array([s1_ts.index[-1], station_ts.index[-1]]).min()
if start > end:
continue
station_ts = station_ts[start:end]
s1_ts = s1_ts[start:end]
gldas_ts = gldas_ts[start:end]
if calcstd == True:
s1_ts_std = s1_ts_std[start:end]
if len(s1_ts) < 1:
continue
s1_ts_res = s1_ts.resample('D').mean()
station_ts_res = station_ts.resample('D').mean()
if calc_anomalies == True:
from pytesmo.time_series import anomaly as pyan
s1_clim = pyan.calc_climatology(s1_ts_res.interpolate())
station_clim = pyan.calc_climatology(
station_ts_res.interpolate())
s1_ts = pyan.calc_anomaly(s1_ts, climatology=s1_clim)
s1_ts_res = pyan.calc_anomaly(s1_ts_res, climatology=s1_clim)
station_ts = pyan.calc_anomaly(station_ts,
climatology=station_clim)
station_ts_res = pyan.calc_anomaly(station_ts_res,
climatology=station_clim)
# calculate error metrics
ts_bias = s1_ts_res.subtract(station_ts_res).mean()
tobemerged = [s1_ts_res.dropna(), station_ts_res.dropna()]
s1_and_station = pd.concat(tobemerged, axis=1, join='inner')
ts_bias = s1_and_station[0].subtract(s1_and_station[1]).median()
xytmp = pd.concat(
{
'y': s1_and_station[0] - ts_bias,
'x': s1_and_station[1]
},
join='inner',
axis=1)
if cntr == 1:
xyplot = xytmp
else:
xyplot = pd.concat([xyplot, xytmp], axis=0)
cntr = cntr + 1
ts_cor = s1_and_station[0].corr(s1_and_station[1])
ts_rmse = np.sqrt(
np.nanmean(
np.square(s1_and_station[0].subtract(s1_and_station[1]))))
ts_ubrmse = np.sqrt(
np.sum(
np.square((s1_and_station[0] - s1_and_station[0].mean()) -
(s1_and_station[1] - s1_and_station[1].mean())))
/ len(s1_and_station[0]))
print('R: ' + str(ts_cor))
print('RMSE: ' + str(ts_rmse))
print('Bias: ' + str(ts_bias))
txtrep.write(st_name + ', ' + str(ts_cor) + ', ' + str(ts_rmse) +
'\n')
s1_ts_list.append(s1_ts)
station_ts_list.append(station_ts)
station_name_list.append(st_name)
gldas_ts_list.append(gldas_ts)
# plot
fig, ax1 = plt.subplots(figsize=(7.16, 1.4), dpi=300)
line1, = ax1.plot(s1_ts.index,
s1_ts,
color='b',
linestyle='-',
marker='+',
label='Sentinel-1',
linewidth=0.2)
line2, = ax1.plot(station_ts.index,
station_ts,
label='In-Situ',
linewidth=0.4)
if np.any(outliers) and outliers is not None:
line6, = ax1.plot(s1_ts.index[outliers],
s1_ts.iloc[outliers],
color='r',
linestyle='',
marker='o')
if calcstd == True:
line4, = ax1.plot(s1_ts.index,
s1_ts - np.sqrt(s1_ts_std),
color='k',
linestyle='--',
linewidth=0.2)
line5, = ax1.plot(s1_ts.index,
s1_ts + np.sqrt(s1_ts_std),
color='k',
linestyle='--',
linewidth=0.2)
line6, = ax1.plot(gldas_ts.index,
gldas_ts,
color='g',
linestyle='--',
label='GLDAS',
linewidth=0.2)
ax1.set_ylabel('Soil Moisture [m3m-3]', size=8)
smc_max = np.max([s1_ts.max(), station_ts.max()])
if smc_max <= 0.5:
smc_max = 0.5
ax1.set_ylim((0, smc_max))
ax1.text(
0.85,
0.4,
'R=' + '{:03.2f}'.format(ts_cor) +
# '\nRMSE=' + '{:03.2f}'.format(ts_rmse) +
'\nBias=' + '{:03.2f}'.format(ts_bias) + '\nubRMSE=' +
'{:03.2f}'.format(ts_ubrmse),
transform=ax1.transAxes,
fontsize=8)
plt.title(st_name, fontsize=8)
plt.tight_layout()
plt.savefig(plotpath, dpi=300)
plt.close()
except:
print('No data for: ' + st_name)
pickle.dump(
(s1_ts_list, station_ts_list, gldas_ts_list, station_name_list),
open(
'C:/Users/FGreifeneder/OneDrive - Scientific Network South Tyrol/1_THESIS/pub3/images_submission2/w_GLDAS_validation_tss_mazia'
+ name + '.p', 'wb'))
urmse_scatter = np.sqrt(
np.sum(
np.square((xyplot['y'] - xyplot['y'].mean()) -
(xyplot['x'] - xyplot['x'].mean()))) / len(xyplot['y']))
rmse_scatter = np.sqrt(
np.nanmean(np.square(xyplot['x'].subtract(xyplot['y']))))
r_scatter = xyplot['x'].corr(xyplot['y'])
# plt.figure(figsize=(3.5, 3), dpi=600)
xyplot.plot.scatter(x='x',
y='y',
color='k',
xlim=(0, 1),
ylim=(0, 1),
figsize=(3.5, 3),
s=1,
marker='.')
plt.xlim(0, 0.7)
plt.ylim(0, 0.7)
plt.xlabel("$SMC_{Tot}$ [m$^3$m$^{-3}$]", size=8)
plt.ylabel("$SMC^*_{Tot}$ [m$^3$m$^{-3}$]", size=8)
plt.plot([0, 0.7], [0, 0.7], 'k--')
plt.text(0.1,
0.5,
'R=' + '{:03.2f}'.format(r_scatter) + '\nRMSE=' +
'{:03.2f}'.format(rmse_scatter),
fontsize=8) # +
# '\nRMSE=' + '{:03.2f}'.format(rmse_scatter), fontsize=8)
plt.tick_params(labelsize=8)
plt.title('True vs. estimated SMC', size=8)
plt.axes().set_aspect('equal', 'box')
plt.tight_layout()
plt.savefig(outpath + '1_Mazia_scatterplot.png', dpi=600)
plt.close()
txtrep.write(
'------------------------------------------------------------------------\n\n'
)
txtrep.write('Overall performance:\n')
txtrep.write('R = ' + str(xyplot['x'].corr(xyplot['y'])) + '\n')
txtrep.write(
'RMSE = ' +
str(np.sqrt(np.nanmean(np.square(xyplot['x'].subtract(xyplot['y']))))))
txtrep.write('ubRMSE = ' + str(
np.sqrt(
np.sum(
np.square((xyplot['y'] - xyplot['y'].mean()) -
(xyplot['x'] - xyplot['x'].mean()))) /
len(xyplot['y']))))
txtrep.close()
