def get_lst_ts(x, y, lst_file):
ds = nc.Dataset(lst_file)
ts = ds['lst'][y, x, :]
ts_sd = ds['lst_sd'][y, x, :]
date = [julday.jul2date2(d).date() for d in ds['date_jul']]
return ts, ts_sd, date
def jul_range(self, jd01, jd02, step=1):
"""
Define output dates.
We want to work with whole years
:param jd01:
:param jd02:
:return: t
"""
year = julday.jul2date2(jd01).year
jd1 = julday.date2jul(dt.date(year, 1, 1))
year = julday.jul2date2(jd02).year
jd2 = julday.date2jul(dt.date(year, 12, 31))
t = np.arange(jd1, jd2 + 1, step)
return t
def get_sar_ts(x, y, sar_file):
ds = nc.Dataset(sar_file)
ts = ds['bs'][y, x, :]
ts_sd = ds['bs_sd'][y, x, :]
date = [julday.jul2date2(d).date() for d in ds['date_jul']]
return ts, ts_sd, date
def get_rho_ts(x, y, year_range, band, rho_file):
rho = []
rho_sd = []
unc = []
jd = []
errors = [5000, 500, 5000, 5000, 5000, 5000]
for year in year_range:
print rho_file % year
ds = nc.Dataset(rho_file % year)
tmp = ds.groups['reflectance'].variables['refl_b%d' % (band)][:, y, x]
try:
tmp = ds.groups['reflectance'].variables['refl_b%d' % (band)][:, y,
x]
rho = np.append(rho, tmp)
except:
print 'year %d is corrupted' % year
ds.close()
ds = nc.Dataset(rho_file % (year - 1))
rho = np.append(
rho,
ds.groups['reflectance'].variables['refl_b%d' % (band)][:, y,
x])
rho_sd0 = ds.groups['reflectance'].variables['refl_b%d_sd' % (band)][:,
y,
x]
jd = np.append(jd, ds.variables['julday'])
err = 5 * np.arange(-1, 1, 2 / float(ds.groups['reflectance']. \
variables['refl_b%d' % (band)][:, y, x].shape[0])) ** 2
unc_0 = 1 / (ds.groups['reflectance'].variables['refl_b%d' %
(band)][:, y, x] *
errors[band]) * err
ind = np.where(
ds.groups['reflectance'].variables['refl_b%d' %
(band)][:, y, x] <= 0)
unc_0[ind] = 1
unc = np.append(unc, unc_0)
rho_sd0[ind] = 1
rho_sd = np.append(rho_sd, rho_sd0)
dates = np.array([julday.jul2date2(d).date() for d in jd])
return rho, unc, dates
def get_rho(data_dir, lon_arr, lat_arr, year_range, band, file_pattern):
"""
Parameters
----------
lon_arr
lat_arr
year_range
band
file_pattern
Returns
-------
"""
rho = []
rho_sd = []
unc = []
dates_fwd = []
dates_orig = []
y_fwd = []
y_orig = []
dates = []
errors = [5000, 500, 5000, 5000, 5000, 5000]
for year in year_range:
print file_pattern % year
ds = nc.Dataset(data_dir + file_pattern % year)
x = np.argmin(np.abs(ds['lon'][:] - lon_arr[0]))
xy = np.unravel_index(x, ds['lon'][:].shape)
print 'lon:', ds['lon'][xy[0], xy[1]]
x = xy[1]
y = np.argmin(np.abs(ds['lat'][:] - lat_arr[0]))
xy = np.unravel_index(y, ds['lat'][:].shape)
print 'lat:', ds['lat'][xy[0], xy[1]]
y = xy[0]
print 'x, y = ', x, y
tmp = ds['reflectance/refl_b%d' % (band + 1)][:, y, x]
# print 'ds[reflectance/refl_b%d][:, %d, %d]: ' % (band+1, y,x), ds['reflectance/refl_b%d' % (band + 1)][:, y, x].shape
# print ds['reflectance/refl_b1']
# print ds['julday']
# try:
tmp = ds['reflectance/refl_b%d' % (band + 1)][:, y, x]
rho = np.append(rho, tmp)
date_tmp = np.array([julday.jul2date2(d).date() for d in ds['julday']])
dates = np.append(dates, date_tmp)
# except:
# print 'year %d is corrupted' % year
# ds.close()
# ds = nc.Dataset(file_pattern % (year - 1))
#
# rho = np.append(rho, ds['reflectance/refl_b%d' % (band + 1)][:, y, x])
# dates = np.append(dates, np.array([julday.jul2date2(d).date() for d in ds['julday']]))
# print 'shapes:', rho.shape, dates.shape
rho_sd0 = ds['reflectance/refl_b%d_sd' % (band + 1)][:, y, x]
err = 5 * np.linspace(
-1, 1, ds['reflectance/refl_b%d' %
(band + 1)][:, y, x].shape[0])**2
unc_0 = 1 / (ds.groups['reflectance'].variables['refl_b%d' %
(band + 1)][:, y, x] *
errors[band]) * err
ind = np.where(
ds.groups['reflectance'].variables['refl_b%d' %
(band + 1)][:, y, x] <= 0)
unc_0[ind] = 1
unc = np.append(unc, unc_0)
rho_sd0[ind] = 1
rho_sd = np.append(rho_sd, rho_sd0)
# print data_dir + 'rho_h18v04_%d_7day_fwd.nc' % year
# ds_fwd = nc.Dataset(data_dir + 'rho_h18v04_%d_7day_fwd.nc' % year)
fname_fwd = file_pattern.split('.')[0] + '_fwd.nc'
print file_pattern.split('/')[-1].split('.')
print fname_fwd
ds_fwd = nc.Dataset(data_dir + fname_fwd % year)
# print ds_fwd['y_fwd']
ind = np.where(ds_fwd['y_fwd'][:, band, y, x] > 0)
y_fwd = np.append(y_fwd, ds_fwd['y_fwd'][:, band, y, x][ind])
dates_fwd = np.append(dates_fwd, date_tmp[ind])
# print 'y_fwd:', y_fwd.shape
ind = np.where(ds_fwd['y_orig'][:, band, y, x] > 0)
y_orig = np.append(y_orig, ds_fwd['y_orig'][:, band, y, x][ind])
dates_orig = np.append(dates_orig, date_tmp[ind])
# print dates_fwd
# print year, rho.shape, dates.shape
return rho, rho_sd, y_fwd, y_orig, dates, dates_fwd, dates_orig
def do_opt(self, obs, obs_sd, date_jul, gamma=70, t=np.arange(1, 365, 1)):
"""
Do optimization for given vector of observation obs
:param obs: vector of observations
:param obs_sd: observational uncertainties
:param date_jul: vector of julian days
:param gamma: dynamical model error (smoothness)
:param t: time grid
:return:
"""
# Define output dates.
# We want to work with whole years
year = julday.jul2date2(date_jul[0]).year
jd1 = julday.date2jul(dt.date(year, 1, 1))
year = julday.jul2date2(date_jul[-1]).year
jd2 = julday.date2jul(dt.date(year, 12, 31))
# t = np.arange(jd1, jd2 + 1, 4)
t_obs = np.arange(jd1, jd2 + 1, 1)
x_prior = np.ones_like(t) * np.mean(obs)
y_obs = np.zeros(t_obs.shape)
sigma_obs = np.zeros(t_obs.shape)
for i in xrange(t_obs.shape[0] - 1):
for j in xrange(date_jul.shape[0]):
if t_obs[i] == date_jul[j]:
y_obs[i] = obs[j]
sigma_obs[i] = obs_sd[j]
obs_mask = np.zeros(t.shape[0]).astype(bool)
obs_mask = np.where(sigma_obs > 0, True, False)
sigma_obs_t = np.zeros(t.shape)
sigma_prior = 100.
retval = np.zeros(x_prior.shape[0])
post_sd = np.zeros(x_prior.shape[0])
# Sometimes fmin_bfgs() can give "input/output error". The reasons are unknown for me. So if this happens we can
# simply return zeros what flags this pixel as a bad one.
try:
retval = scipy.optimize.fmin_bfgs(jc.cost_function_t, x_prior, fprime=jc.der_cost_function_t, \
args=(y_obs, x_prior, sigma_obs, sigma_prior, gamma, obs_mask, t, t_obs),
disp=1, retall=1)[0]
except:
print 'An error in fmin_bfgs():', sys.exc_info()[0]
return retval, post_sd, sigma_prior, sigma_obs_t
# Calculate observational uncertainty in grid of output.
# This is requared for calculation of posteriour uncertainty
for i in xrange(t.shape[0] - 1):
ind = np.logical_and(t_obs >= t[i], t_obs < t[i + 1])
k = 0
for j in xrange(t_obs[ind].shape[0]):
if y_obs[ind][j] != 0:
sigma_obs_t[i] = sigma_obs_t[i] + sigma_obs[ind][j]
k += 1
if k != 0:
sigma_obs_t[i] = sigma_obs_t[i] / k
# der_j_obs[i] = (x[i] - y[ind][j]) ** 2 / sigma_obs[ind][j] ** 2
# We assume that uncertainty of no observation is very large
sigma_obs_t[sigma_obs_t == 0] = 999
# pdb.set_trace()
# Do uncertainty
post_sd = jc.posterior_covariance(retval, sigma_prior, sigma_obs_t,
gamma)
return retval, post_sd, sigma_prior, sigma_obs_t
def do_job(self, ncd_out, downsample_dir, file_pattern, year, tile, ulx, uly, lrpx, lrpy, cc, rr, step=1):
print 'Starting doing main job with ' + ncd_out
site =''
#f_list = np.array([file for file in glob.glob(downsample_dir + sent1_mod_file)])
file_in_1 = file_pattern % (year, tile)
# get backscatter for the current year for block (ulx, uly) with size cc/rr (columns rows)
sent1_in, sent1_in_sd, date_jul, date = self.get_back(downsample_dir, file_in_1, ulx, uly, cc, rr)
# get first and last julian days for 'year'
year = julday.jul2date2(date_jul[0]).year
d1 = julday.date2jul(dt.date(year, 1, 1))
d2 = julday.date2jul(dt.date(year, 12, 31))
# get temporal range
self.t = np.arange(d1, d2 + 1, 1)
t_out = np.arange(d1, d2 + 1, step)
head_flag = False
tail_flag = False
# get filename for previous year
file_in_0 = file_pattern % (year - 1, tile)
f_list = np.array([file for file in glob.glob(downsample_dir + file_in_0)])
# if data exist get backscatter for previous year
if len(f_list) != 0:
sent1_in_0, sent1_in_0_sd, date_jul_0, date_0 = self.get_back(downsample_dir, file_in_0, ulx, uly, cc, rr)
# get data if we have any good pixels
if np.max(sent1_in_0[:]) != -99:
# we need last day for this year therefore looping backward
for i in xrange(sent1_in_0.shape[0]-1, 0, -1):
if np.max(sent1_in_0[i, :, :]) != -99:
tmp = np.zeros((sent1_in.shape[0]+1, sent1_in.shape[1], sent1_in.shape[2]))
tmp[1:,:, :] = sent1_in[:]
tmp[0, :, :] = sent1_in_0[i, :, :]
sent1_in = tmp.copy()
tmp[1:,:, :] = sent1_in_sd[:]
tmp[0, :, :] = sent1_in_0_sd[i,:,:]
sent1_in_sd = tmp.copy()
head_flag = True
break
# add first date
date_jul = np.append(d1, date_jul)
date = np.append(dt.date(year, 1, 1), date)
# get filename for next year
file_in_2 = file_pattern % (year + 1, tile)
f_list = np.array([file for file in glob.glob(downsample_dir + file_in_2)])
if len(f_list) != 0:
sent1_in_2, sent1_in_2_sd, date_jul_2, date_2 = self.get_back(downsample_dir, file_in_2, ulx, uly, cc, rr)
if np.max(sent1_in_2[:]) != -99:
for i in xrange(0, sent1_in_2.shape[2]):
if np.max(sent1_in_2[i, :, :]) != -99:
tmp = np.zeros((sent1_in.shape[0]+1, sent1_in.shape[1], sent1_in.shape[2]))
tmp[:-1,:, :] = sent1_in[:]
tmp[-1, :, :] = sent1_in_2[i, :, :]
sent1_in = tmp.copy()
tmp[:-1, :, :] = sent1_in_sd[:]
tmp[-1, :, :] = sent1_in_2_sd[i, :, :]
sent1_in_sd = tmp.copy()
tail_flag = True
break
# add last date
date_jul = np.append(date_jul, d2)
date = np.append(date, dt.date(year, 12, 31))
#t = np.arange(date_jul[0] - 1, date_jul[-1] + 1)
sent1_out = np.zeros((t_out.shape[0], sent1_in.shape[1], sent1_in.shape[2]))
sent1_out_sd = np.zeros((t_out.shape[0], sent1_in.shape[1], sent1_in.shape[2]))
if np.max(sent1_in) != -99:
for i in xrange(sent1_in.shape[1]):
for j in xrange(sent1_in.shape[2]):
#print 'pixel %d %d' % (i,j)
# retval, post_sd = self.do_opt2(sent1_in[i, j, :], sent1_in_sd[i, j, :], date_jul, date, t_out)
retval, post_sd, sigma_prior, sigma_obs_t = self.do_opt3(sent1_in[:, i, j],\
sent1_in_sd[:, i, j],\
date_jul, gamma=1, t=t_out)
# sent1_out[i, j, :] = 10 * np.log10(retval)
# sent1_out_sd[i, j, :] = 10 * np.log10(post_sd)
sent1_out[:, i, j] = retval
sent1_out_sd[:, i, j] = post_sd
if head_flag == True:
sent1_in = np.delete(sent1_in, 0, axis=0)
sent1_in_sd = np.delete(sent1_in_sd, 0, axis=0)
date_jul = np.delete(date_jul, 0)
date = np.delete(date, 0)
if tail_flag == True:
sent1_in = np.delete(sent1_in, -1, axis=0)
sent1_in_sd = np.delete(sent1_in_sd, -1, axis=0)
date_jul = np.delete(date_jul, -1)
date = np.delete(date, -1)
print 'sent1_in:', sent1_in.shape
# self.save_netcdf(sent1_out, sent1_out_sd, self.t, 'sentinel/sent1_%d_%d_%d_%d.nc' % (ulx, uly, cc, rr))
#ind = np.in1d(self.t, date_jul)
ind = []
for ii, d0 in enumerate(self.t):
for d1 in date_jul:
if d0 == d1:
ind = np.append(ind, ii).astype(int)
bs_orig = np.ones(sent1_in.shape) * 10000
bs_orig_sd = np.ones(sent1_in.shape) * 10000
# bs_orig[:,:,ind] = sent1_in
# bs_orig_sd[:, :, ind] = sent1_in_sd
bs_orig[:] = sent1_in
bs_orig_sd[:] = sent1_in_sd
bs_orig[np.isnan(bs_orig)] = 10000
bs_orig_sd[np.isnan(bs_orig_sd)] = 10000
# Save result to netCDF file
self.save_netcdf(bs_orig, bs_orig_sd, date_jul, sent1_out, sent1_out_sd, t_out, ncd_out)
print 'run_job series is done!!!'
do_plot = False
if do_plot:
i = 0
j = 0
# t = np.arange(date_jul[0] - 1, date_jul[-1] + 1)
ci_5 = np.sqrt(2) * ssp.erfinv(0.05)
ci_25 = np.sqrt(2) * ssp.erfinv(0.25)
ci_75 = np.sqrt(2) * ssp.erfinv(0.75)
plt.fill_between(t_out, sent1_out[:, i, j] - ci_75 * post_sd, sent1_out[:, i, j] + ci_75 * post_sd,
color='0.9')
plt.fill_between(t_out, sent1_out[:, i, j] - ci_25 * post_sd, sent1_out[:, i, j] + ci_25 * post_sd,
color='0.8')
plt.fill_between(t_out, sent1_out[:, i, j] - ci_5 * post_sd, sent1_out[:, i, j] + ci_5 * post_sd,
color='0.6')
# plt.fill_between(t, retval[0])
print i,j
plt.plot(t_out, sent1_out[:, i, j], label='Interpolated backscatter')
# plt.errorbar(date_jul, 10 * np.log10(sent1_in[i, j, :]), yerr=10 * np.log10(sent1_in_sd[i, j, :]),
# marker='o', ls='')
ind = sent1_in[:, i, j] > -90
try:
plt.errorbar(date_jul[ind], sent1_in[ind, i, j], yerr=sent1_in_sd[ind, i, j],
marker='o', ls='', label='Observations')
except:
print 'can not plot errorbar. sorry.'
#plt.ylim(-5, -15)
plt.xlabel('Julian day', fontsize='large')
plt.ylabel('Backscatter', fontsize='large')
plt.legend(loc=0, fontsize='large')
plt.show()
def do_opt2(self, back_scat, back_scat_sd, date_jul, date, t = np.arange(1, 365, 1)):
"""
Find optimal smooth solution for time series of backsctter data
taking uncertainties into account.
:param back_scat:
:param back_scat_sd:
:param date_jul:
:param date:
:return:
"""
# back_scat = bs
# back_scat = bs_sd
# date_jul = dj_opt
# date = d_opt
year = julday.jul2date2(date_jul[0]).year
d1 = julday.date2jul(dt.date(year, 1, 1))
year = julday.jul2date2(date_jul[-1]).year
d2 = julday.date2jul(dt.date(year, 12, 31))
x = np.arange(d1, d2 + 1)
ind = np.logical_and(~np.isnan(back_scat), back_scat > -90)
back_scat = back_scat[ind]
back_scat_sd = back_scat_sd[ind]
date_jul = date_jul[ind]
date = date[ind]
x_prior = np.ones_like(x) * np.mean(back_scat)
y_obs = np.zeros(x.shape)
sigma_obs = np.zeros(x.shape)
sigma_obs[:] = 99
for i in xrange(x.shape[0]):
for j in xrange(date_jul.shape[0]):
if x[i] == date_jul[j]:
y_obs[i] = back_scat[j]
sigma_obs[i] = back_scat_sd[j]
obs_mask = np.ones(x.shape[0]).astype(int)
obs_mask = np.where(y_obs != 0, True, False)
sigma_prior = 100.
gamma = 1
# Do optimization of sentinel data in order to restore gaps
# print 'starting fmin_bfgs with y_obs ', y_obs[obs_mask].shape
# plt.plot(y_obs[obs_mask])
# plt.plot(x_prior[obs_mask])
# plt.show()
# retval = scipy.optimize.fmin_bfgs(jc.cost_function, x_prior, fprime=jc.der_cost_function, \
# args=(y_obs[obs_mask], x_prior, sigma_obs[obs_mask], sigma_prior, gamma,
# obs_mask), disp=1, retall=15, maxiter=1500)
bounds = np.zeros((x_prior.shape[0], 2))
bounds[:, 0] = -200
bounds[:, 1] = 0.5
if y_obs[obs_mask] != []:
retval = scipy.optimize.fmin_l_bfgs_b(jc.cost_function, x_prior, fprime=jc.der_cost_function, \
args=(y_obs[obs_mask], x_prior, sigma_obs[obs_mask], sigma_prior, gamma,
obs_mask),\
bounds = bounds, factr=10000000, pgtol=1e-05, maxiter=1500,\
iprint=0)[0]
# disp=0,
# Do uncertainty
try:
post_sd = jc.posterior_covariance(retval, sigma_prior, sigma_obs, gamma, obs_mask)
except:
print 'jc.posterior_covariance exception'
post_sd = np.ones(x.shape[0])
else:
retval = np.ones(x.shape[0])
post_sd = np.ones(x.shape[0])
ind = []
for jj in xrange(x.shape[0]):
if x[jj] in self.t:#x0:
ind = np.append(ind, jj).astype(int)
return retval[ind], post_sd[ind]
def do_opt(self, bs, bs_sd, dj_opt, d_opt):
"""
Find optimal smooth solution for time series of backsctter data
taking uncertainties into account.
:param back_scat:
:param back_scat_sd:
:param date_jul:
:param date:
:return:
"""
# bs = np.concatenate((sent1_in[i, j, :], sent1_in[i, j, :], sent1_in[i, j, :]))
# bs_sd = np.concatenate((sent1_in_sd[i, j, :], sent1_in_sd[i, j, :], sent1_in_sd[i, j, :]))
# dj_opt = np.concatenate((date_jul_1, date_jul, date_jul_2))
# d_opt = np.concatenate((date_1, date, date_2))
# date0 = []
#x0 = np.arange(dj_opt[0] - 1, dj_opt[-1] + 1)
dj_opt1 = []
dj_opt2 = []
d_opt1 = []
d_opt2 = []
for jd in dj_opt:
d = julday.jul2date2(jd).date()
# print d
d0 = datetime.date(d.year - 1, d.month, d.day)
d_opt1 = np.append(d_opt1, d0)
# print date0[-1]
dj_opt1 = np.append(dj_opt1, julday.date2jul(d0))
d0 = datetime.date(d.year + 1, d.month, d.day)
d_opt2 = np.append(d_opt2, d0)
dj_opt2 = np.append(dj_opt2, julday.date2jul(d0))
# date_1 = np.append(date_1, datetime.date())
back_scat = np.concatenate((bs, bs, bs))
back_scat_sd = np.concatenate((bs_sd, bs_sd, bs_sd))
date_jul = np.concatenate((dj_opt1, dj_opt, dj_opt2))
date = np.concatenate((d_opt1, d_opt, d_opt2))
# back_scat = bs
# back_scat_sd = bs_sd
# date_jul = dj_opt
# date = d_opt
year = julday.jul2date2(date_jul[0]).year
d1 = julday.date2jul(dt.date(year, 1, 1))
year = julday.jul2date2(date_jul[-1]).year
d2 = julday.date2jul(dt.date(year, 12, 31))
x = np.arange(d1, d2 + 1)
#x = np.arange(date_jul[0] - 1, date_jul[-1] + 1)
ind = ~np.isnan(back_scat)
back_scat = back_scat[ind]
back_scat_sd = back_scat_sd[ind]
date_jul = date_jul[ind]
date = date[ind]
x_prior = np.ones_like(x) * np.mean(back_scat)
# print x
y_obs = np.zeros(x.shape)
sigma_obs = np.zeros(x.shape)
for i in xrange(x.shape[0]):
for j in xrange(date_jul.shape[0]):
if x[i] == date_jul[j]:
y_obs[i] = back_scat[j]
sigma_obs[i] = back_scat_sd[j]
obs_mask = np.ones(x.shape[0]).astype(int)
obs_mask = np.where(y_obs > 0, True, False)
# print 'obs_mask\n', obs_mask
# print 'y_obs\n', y_obs
sigma_prior = 100.
gamma = 1
# sigma_obs = 0.015
# os.chdir('/home/ucfamc3/max_python/baci_brdf/')
# Do optimization of sentinel data in order to restore gaps
print 'starting fmin_bfgs with y_obs ', y_obs[obs_mask].shape
retval = scipy.optimize.fmin_bfgs(jc.cost_function, x_prior, fprime=jc.der_cost_function, \
args=(y_obs[obs_mask], x_prior, sigma_obs[obs_mask], sigma_prior, gamma,
obs_mask), disp=1, retall=15, maxiter=1500)
bounds = bounds = np.zeros((x_prior.shape[0], 2))
bounds[:, 0] = 0
bounds[:, 1] = 0.5
# retval = scipy.optimize.fmin_l_bfgs_b(jc.cost_function, x_prior, fprime=jc.der_cost_function, \
# args=(y_obs[obs_mask], x_prior, sigma_obs[obs_mask], sigma_prior, gamma,
# obs_mask),\
# bounds = bounds, factr=10000000, pgtol=1e-05, maxiter=1500, disp=1)
# retval = scipy.optimize.minimize(jc.cost_function, x_prior, method="L-BFGS-B", \
# jac=True, bounds=the_bounds, options=self.optimisation_options)
# retval = scipy.optimize.minimize(jc.cost_function, x_prior, method="L-BFGS-B", \
# args=(y_obs[obs_mask], x_prior, sigma_obs[obs_mask], sigma_prior, gamma, obs_mask),\
# jac=False)
#
# self.optimisation_options = {"factr": 1000, \
# "m": 400, "pgtol": 1e-12, "maxcor": 200, \
# "maxiter": 1500, "disp": True}
# Do uncertainty
post_sd = jc.posterior_covariance(retval[0], sigma_prior, sigma_obs, gamma, obs_mask)
ind = []
for jj in xrange(x.shape[0]):
if x[jj] in self.t:#x0:
ind = np.append(ind, jj).astype(int)
return retval[0][ind], post_sd[ind]