def get_img_air(self, f_name, px, py, cols, rows, sub_ds_n=0):
"""
get an array from a geo image.
Parameters
-----------
f_name : str
file name
px : int
x pixel
py : int
y pixel
cols : int
number of columns
rows : int
number of rows
sub_ds_n : int
number of a subdataset
"""
# print 'f_name:', f_name
# print np.load(f_name).keys()
img = np.load(f_name)['air_new_interp'][px:px + cols, py:py + rows]
d = int(f_name.split('.')[0].split('_')[-1][1:])
y = int(f_name.split('.')[0].split('_')[-2])
jd = julday.date2jul(julday.doy2date(y, d))
return img, jd
def jul_range(self, jd01, jd02, step=1):
"""
Define output dates.
We want to work with whole years
:param jd01:
:param jd02:
:return: t
"""
year = julday.jul2date2(jd01).year
jd1 = julday.date2jul(dt.date(year, 1, 1))
year = julday.jul2date2(jd02).year
jd2 = julday.date2jul(dt.date(year, 12, 31))
t = np.arange(jd1, jd2 + 1, step)
return t
def CalcSun(lat, lon, date, time=12./24., zone=-6):
#zone - time zone (+ to E)
#jd = 2452276.25
#sun_dec - Sun Declin (deg)
#hour_angle - Hour Angle (deg)
#tst - True Solar Time (min)
#et - Eq of time (min)
#var_y
#gml - Geom Mean Long Sun (deg)
#eeo - Eccent Earth Orbit
#gma - Geom Mean Anom Sun (deg)
#jc - Julian Century
#oc - Obliq Corr (deg)
#moe - Mean Obliq Ecliptic (deg)
#sal - Sun App Long (deg)
#stl - Sun True Long (deg)
#sec - Sun Eq of Ctr
jd = np.round(julday.date2jul(date))
jc = (jd-2451545.)/36525.
gma = 357.52911+jc*(35999.05029 - 0.0001537*jc)
gml = (280.46646+jc*(36000.76983 + jc*0.0003032))%360
eeo = 0.016708634-jc*(0.000042037+0.0000001267*jc)
moe = 23+(26+((21.448-jc*(46.815+jc*(0.00059-jc*0.001813))))/60.)/60
oc = moe+0.00256*np.cos(radians(125.04-1934.136*jc))
var_y = np.tan(radians(oc/2.))*np.tan(radians(oc/2.))
sec = np.sin(radians(gma))*(1.914602-jc*(0.004817+0.000014*jc)) + np.sin(radians(2*gma))*(0.019993-0.000101*jc)+np.sin(radians(3*gma))*0.000289
stl = gml + sec
sal = stl-0.00569-0.00478*np.sin(radians(125.04-1934.136*jc))
sun_dec = degrees(np.arcsin(np.sin(radians(oc))*np.sin(radians(sal))))
et = 4.*degrees(var_y*np.sin(2.*radians(gml)) - 2.*eeo*np.sin(radians(gma)) + 4.*eeo*var_y*np.sin(radians(gma))*np.cos(2*radians(gml)) -\
0.5*var_y*var_y*np.sin(4.*radians(gml)) - 1.25*eeo*eeo*np.sin(2.*radians(gma)))
tst = (time*1440.+et+4*lon-60.*zone)%1440.
if tst/4. < 0:
hour_angle = tst/4.+180
else:
hour_angle = tst/4.-180
theta_i = degrees(np.arccos(np.sin(radians(lat))*np.sin(radians(sun_dec)) + np.cos(radians(lat))*np.cos(radians(sun_dec))*np.cos(radians(hour_angle))))
if hour_angle>0:
phi_i = (degrees(np.arccos(((np.sin(radians(lat))*np.cos(radians(theta_i)))-np.sin(radians(sun_dec)))/(np.cos(radians(lat))*np.sin(radians(theta_i)))))+180)%360.
else:
phi_i = (540-degrees(np.arccos(((np.sin(radians(lat))*np.cos(radians(theta_i)))-np.sin(radians(sun_dec)))/(np.cos(radians(lat))*np.sin(radians(theta_i))))))%360.
return theta_i, phi_i
def get_obs_operator_etm(emu_file, state, year, ind_cross, state_file, cost_weight=1):
wl_misr = [483.0, 560.0, 662.0, 835.0, 1648.0, 2206.0]
wl_width = [65, 80, 60, 150, 200, 270]
wl_full = np.arange(400, 2501)
# Load data a file
d = np.loadtxt(state_file, skiprows=1)
d = d[ind_cross, :]
jul = d[:, 0].astype(int)
dt = datetime
# Find difference between first day of an year and array of all julian days
day_diff = jul - julday.date2jul(dt.date(year, 1, 1))
# Find days for this year
ind = np.logical_and(day_diff > 0, day_diff < 366)
# Find unique days in an year. Landsat data can contain two observations for one day.
# This is not supported in the new generation...
(arr, ind_u) = np.unique(jul, return_index=True)
# array of not unique indices
ind_z = np.zeros(jul.shape[0], dtype=bool)
ind_z[:] = True
# set all unique indices to False
ind_z[ind_u] = False
# set all not unique indices to False
ind[ind_z] = False
doys = []
for day1 in jul[ind]:
doys = np.append(doys, julday.jul2doy(day1))
print "doys=", doys
vza = np.round(d[ind, 2])
sza = np.round(d[ind, 4])
vaa = np.round(d[ind, 3])
saa = np.round(d[ind, 5])
raa = np.round(abs(vaa - saa))
rho = d[ind, 6:]
# print 'vza=', vz
print "rho.shape (etm):", rho.shape
# 446nm +-21, 558nm +-15, 672nm +-11, 866nm +-20
b_min = np.array(wl_misr) - np.array(wl_width)
b_max = np.array(wl_misr) + np.array(wl_width)
n_bands = b_min.shape[0]
band_pass = np.zeros((n_bands, 2101), dtype=np.bool)
bw = np.zeros(n_bands)
bh = np.zeros(n_bands)
for i in xrange(n_bands):
band_pass[i, :] = np.logical_and(wl_full >= b_min[i], wl_full <= b_max[i])
bw[i] = b_max[i] - b_min[i]
bh[i] = (b_max[i] + b_min[i]) / 2.0
sigma_min = 0.001
sigma_max = 0.04
sigma_obs = (sigma_max - sigma_min) * (bh - bh.min()) / (bh.max() - bh.min())
sigma_obs += sigma_min
bu = sigma_obs
print bu
bu = [0.0041, 0.0045, 0.0047, 0.0053, 0.0079, 0.0097]
print bu
rho_big = np.zeros((365, n_bands))
mask = np.zeros((365, 4))
for i in state.state_grid:
if i in doys:
r = rho[doys == i, :].squeeze()
print doys
rho_big[i - 1, :] = r
# mask[ i, :] = [ i, raa[doys==i], np.round(sza[doys==i]/5.)*5, np.round(vza[doys==i]/5.)*5 ]
mask[i - 1, :] = [i, raa[doys == i], np.round(sza[doys == i]), np.round(vza[doys == i])]
ind = np.where(mask != 0)
emulators = {}
for i in range(doys.shape[0]):
tag = tuple([np.round(sza[i]), np.round(vza[i])])
file_emu = emu_file % (sza[i], vza[i], raa[i])
print sza[i], vza[i], raa[i]
if os.path.isfile(file_emu):
print file_emu
print tag
emulators[tag] = MultivariateEmulator(dump=file_emu)
else:
print "emulator file " + file_emu + " does not exist"
print sza[i], vza[i], raa[i]
print vaa[i], saa[i]
obs = ObservationOperatorMax(state.state_grid, state, rho_big, mask, emulators, bu, band_pass, bw, cost_weight)
return obs, doys
def do_average(file_in, file_out, year, step=30, ref_file='', meter_file=''):
"""
get an array from a geo image.
Parameters
-----------
f_name : str
file name
px : int
x pixel
py : int
y pixel
cols : int
number of columns
rows : int
number of rows
sub_ds_n : int
number of a subdataset
"""
print 'Read input file:'
ds_in = h5py.File(file_in, 'r')
# ds_out = nc.Dataset(path_out + 'lst_average_%s_%d_%dday.nc' % (tile, year, step), 'w')
ds_out = nc.Dataset(file_out, 'w')
ds_out.createDimension('x', 1200)
ds_out.createDimension('y', 1200)
# date_step = np.arange(1, ds_in['doy'].shape[0] + 1, step)
date_step = np.arange(1, 366, step)
ds_out.createDimension('time', date_step.shape[0])
print 'Create variables:'
lstVar = ds_out.createVariable('lst', 'f8', ('time', 'x', 'y'), zlib=True)
lstVar.standard_name = 'LST'
lstVar.long_name = 'Land Surface Temperature'
lstVar.units = 'K'
ds_out.createVariable('lst_sd', 'f8', ('time', 'x', 'y'), zlib=True)
lstVar.standard_name = 'LST SD'
lstVar.long_name = 'Land Surface Temperature Uncertainty'
lstVar.units = 'K'
timeVar = ds_out.createVariable('time', 'f8', 'time')
timeVar.units = 'days since -4712-01-01 00:00:00'
xVar = ds_out.createVariable('x', 'f8', ('x'), zlib=True)
xVar.standard_name = 'projection_x_coordinate'
xVar.long_name = 'x distance on the projection plane from the origin'
xVar.units = 'm'
yVar = ds_out.createVariable('y', 'f8', ('y'), zlib=True)
yVar.standard_name = 'projection_y_coordinate'
yVar.long_name = 'y distance on the projection plane from the origin'
yVar.units = 'm'
meters = np.load(meter_file)
xVar[:] = meters['mx']
yVar[:] = meters['my']
lonVar = ds_out.createVariable('lon', 'f8', ('y', 'x'), zlib=True)
latVar = ds_out.createVariable('lat', 'f8', ('y', 'x'), zlib=True)
lonVar.long_name = "longitude"
lonVar.units = 'degrees_east'
latVar.long_name = "latitude"
latVar.units = 'degrees_north'
# lat = np.zeros((yVar.shape[0], xVar.shape[0]))
# lon = np.zeros((yVar.shape[0], xVar.shape[0]))
# get lat-lon by projected (map) coordinates
print 'Get Lat/Lon:'
for i in xrange(xVar.shape[0]):
for j in xrange(yVar.shape[0]):
# print 'xVar[i], yVar[j]', xVar[i], yVar[j]
latVar[j, i], lonVar[j, i] = get_latlon(xVar[i], yVar[j])
# lonVar[:] = dsin['lon'][:]
# latVar[:] = dsin['lat'][:]
print 'Create crs:'
crs = ds_out.createVariable('crs', 'S1')
crs.grid_mapping_name = 'sinusoidal'
crs.longitude_of_central_meridian = '0.0'
crs.longitude_of_prime_meridian = '0.0'
crs.semi_major_axis = '6371007.181'
crs.false_easting = '0.0'
crs.false_northing = '0.0'
crs.earth_radius = '6371.229'
gds = gdal.Open(ref_file)
geo_ref = gds.GetGeoTransform()
crs.spatial_ref = gds.GetProjection()
crs.GeoTransform = gds.GetGeoTransform()
print 'Do average:'
for i, d in enumerate(date_step):
ind = np.logical_and(ds_in['doy'][:] >= d, ds_in['doy'][:] <
(d + step))
if np.max(ind) == True:
img = ds_in['lst'][ind, :, :] * 0.02
img[img == 0] = np.NaN
ds_out['lst'][i, :, :] = np.nanmean(img, axis=0)
ds_out['lst_sd'][i, :, :] = np.nanstd(img, axis=0)
else:
ds_out['lst'][i, :, :] = np.NaN
ds_out['lst_sd'][i, :, :] = np.NaN
ds_out['lst'][i, :, :][np.isnan(ds_out['lst'][i, :, :])] = ma.masked
ds_out['time'][i] = julday.date2jul(julday.doy2date(year, d))
return 0
def do_job(self, ncd_file, mod11_dir, air_dir, mod11_file, site, year0, tile, ulpx, ulpy, lrpx, lrpy, cc, rr,\
do_cross=True, do_plot = True, step=1):
"""
Run regularization for a block with size (cc x rr) of a MOD11 image
:param mod11_dir: str; Directory of MOD11 files
:param mod11_file: str; Pattern for files search
:param ulpx: int; upper left x pixel of a block
:param ulpy: int; upper left y pixel of a block
:param cc: int; number of columns
:param rr: int; number o rows
:return:
"""
print 'ulpx, ulpy:', ulpx, ulpy
# Get lists of files of a current year and +/-
# f_list = []
# f_list1 = np.array([file for file in glob.glob(mod11_dir + 'MOD11A1.A%d*.hdf' % (year0 - 1))])
# f_list2 = np.array([file for file in glob.glob(mod11_dir + mod11_file)])
# f_list3 = np.array([file for file in glob.glob(mod11_dir + 'MOD11A1.A%d*.hdf' % (year0 + 1))])
#
# f_list = np.concatenate((f_list1, f_list2, f_list3))
# f_list = f_list2
# get file lists of air temperature
# air_dir = '/group_workspaces/cems/baci/sigil/air_temperature/1_km/'
f_list_air = []
f_list_air_1 = np.array([
file for file in glob.glob(air_dir + 'air_new_%s_%d_d*.npz' %
(tile, (year0 - 1)))
])
f_list_air_2 = np.array([
file for file in glob.glob(air_dir + 'air_new_%s_%d_d*.npz' %
(tile, year0))
])
f_list_air_3 = np.array([
file for file in glob.glob(air_dir + 'air_new_%s_%d_d*.npz' %
(tile, (year0 + 1)))
])
def sort_files(flst):
"""
sort filesnames in flst in ascending order
:param flst:
:return:
"""
days = []
for ss in flst:
days = np.append(
days,
ss.split('/')[-1].split('_')[-1].split('.')[0].split('d')
[-1]).astype(int)
ind = np.argsort(days)
flst = flst[ind]
return flst
f_list_air_1 = sort_files(f_list_air_1)
f_list_air_2 = sort_files(f_list_air_2)
f_list_air_3 = sort_files(f_list_air_3)
# if we don't have first or last year, assert an empty list to middle year
f_list_air = f_list_air_2
if f_list_air_1.shape[0] == 0:
f_list_air_1 = f_list_air_2
if f_list_air_3.shape[0] == 0:
f_list_air_3 = f_list_air_2
# make a range of julian days from first to last day of year
jd0 = julday.date2jul(julday.doy2date(year0, 1))
n_days, date_range = self.get_n_days(year0)
jd1 = julday.date2jul(julday.doy2date(year0, n_days))
# input temporal grid (jul days)
t = self.jul_range(jd0, jd1, step=1)
# output temporal grid with specific step
t_out = self.jul_range(jd0, jd1, step=step)
fname = mod11_dir + mod11_file % year0
fname1 = mod11_dir + mod11_file % (year0 - 1)
fname2 = mod11_dir + mod11_file % (year0 + 1)
img, img_qc, doys = self.get_img(fname, ulpx, ulpy, cc, rr, sub_ds_n=0)
# LST array
# img = np.zeros((t.shape[0], cc, rr))
# img_qc = np.zeros(img.shape).astype(int)
img_out = np.zeros((t_out.shape[0], img.shape[1], img.shape[2]))
img_out_sd = np.zeros(img_out.shape)
# Go to Kelvins
img = img * 0.02
img_qc = img_qc.astype(int)
# img[doys-1, :, :] = img0 * 0.02
# img_qc[doys - 1, :, :] = img_qc0
# jul_days = np.zeros(t.shape[0])
jul_days = []
for doy in doys:
# jul_days[doy - 1] = julday.date2jul(julday.doy2date(year0, doy))
jul_days = np.append(jul_days,
julday.date2jul(julday.doy2date(year0, doy)))
jul_days[0] = jd0
jul_days[-1] = jd1
# array for air temperature
# img_air = np.zeros((t.shape[0], cc, rr))
img_air = np.zeros((f_list_air.shape[0], cc, rr))
jd_air = []
for i, fname in enumerate(f_list_air):
data_tmp, dd = self.get_img_air(fname,
ulpx,
ulpy,
cc,
rr,
sub_ds_n=0)
# img_air[dd - 1, :, :] = data_tmp
img_air[i, :, :] = data_tmp
jd_air = np.append(jd_air, dd)
# copy temperature from last day of previous year where QC is good
if os.path.isfile(fname1):
print 'copy temperature from last day of previous year where QC is good'
ii = -1
qc0 = 2
# tmp_qc = self.get_img(fname1, ulpx, ulpy, cc, rr, sub_ds_n=1).astype(int)
tmp_img, tmp_qc, tmp_doys = self.get_img(fname1,
ulpx,
ulpy,
cc,
rr,
sub_ds_n=0)
while np.logical_or(qc0 == 2, qc0 == 3):
# tmp = self.get_img(f_list1[ii], ulpx, ulpy, cc, rr, sub_ds_n=1).astype(int)
qc0 = tmp_qc[ii, cc / 2, rr / 2]
# img[:, :, 0] = self.get_img(f_list1[ii], ulpx, ulpy, cc, rr, sub_ds_n=0) * 0.02
img[0, :, :] = tmp_img[ii, :, :] * 0.02
img_air[0, :, :], jd = self.get_img_air(f_list_air_1[ii],
ulpx,
ulpy,
cc,
rr,
sub_ds_n=0)
img_qc[0, :, :] = tmp_qc[ii, :, :]
ii = ii - 1
if ii * (-1) >= tmp_img.shape[0]:
break
img_orig = img[:].copy()
# f_list1 = np.array([])
# f_list3 = np.array([])
# LST uncertainty
img_err = self.get_lst_unc(img_qc)
# Make an array of uncertainties
# ind = img_air == 0
for ii in xrange(jd_air.shape[0]):
if np.logical_and(0 in img_air[ii, :, :], jd_air[ii] in jul_days):
img_err[jul_days == jd_air[ii], :, :] = 999
ind = img_orig == 0
img_err[ind] = 999
# ind = img != 0
ind = img_err != 999
img_diff = np.zeros(img.shape)
ind_air = []
for ii, jj in enumerate(jul_days):
if jj in jd_air:
img_diff[
ii, :, :] = img_air[jd_air == jj, :, :] - img[ii, :, :]
ind_air = np.append(ind_air, jj)
else:
print jj
# Go to difference between LST and air temperature
# img = img_air - img
# remove water etc from input data
ind_err = img_err == 999
img_diff[ind_err] = 0
# find uncertaities of LST and air difference as percentage of LST uncertainties.
perc = (img_err * 100) / (np.max(img_orig[ind]) -
np.min(img_orig[ind]))
# img_err_air = ((np.max(img[ind]) - np.min(img[ind])) * perc) / 100.
img_err_air = img_err.copy()
#if np.min(img_err) == 999:
#if img.shape[0] > 0:
#img_err = np.zeros(img.shape)
# print 'no data in this block\n'
# self.save_netcdf(np.moveaxis(img_out, 0, 2), \
# np.moveaxis(img_out_sd, 0, 2), \
# np.moveaxis(img_orig, 0, 2), \
# np.moveaxis(img_err, 0, 2), \
# t, jul_days, ncd_file, geo, proj)
# return -1
cx = img_diff.shape[1] / 2
cy = img_diff.shape[2] / 2
if do_cross:
# Cross validation
img_tmp = img_diff.copy()
img_tmp[img_err == 999] = np.NaN
img_mean = np.nanmean(img_tmp, axis=(1, 2))
ind = ~np.isnan(img_mean)
#ind = img_err[:, cx, cy] != 999
#y_obs = img[ind, cx, cy]
y_obs = img_mean[ind]
# sigma_obs = img_err[ind, cx, cy]
# sigma_obs = img_err[ind, cx, cy]
sigma_obs_air = img_err_air[ind, cx, cy]
date_obs = jul_days[ind]
# num_off = y_obs.shape[0] * (1 - (30 / float(y_obs.shape[0])))
num_off = int(y_obs.shape[0] * 0.7)
ind_cv = np.unique(
np.round(np.random.rand(num_off) *
(y_obs.shape[0] - 1)).astype(int))
diff = []
# gammas = [0.1, 0.5, 1, 10, 20, 50, 70, 100, 200, 500, 1000, 1500, 2000, 3000]
gammas = [1, 10, 20, 50, 70, 100, 200, 500, 1000, 1500, 2000, 3000]
for gamma in gammas:
mask = np.ones(y_obs.shape).astype(bool)
mask[ind_cv] = False
print 'gamma=', gamma
retval_0, post_sd_0, sigma_prior, sigma_obs_t = self.do_opt(y_obs[ind_cv], sigma_obs_air[ind_cv],\
date_obs[ind_cv], gamma=gamma, t=t_out)
ind_in = np.in1d(t, date_obs[mask])
try:
diff = np.append(
diff, np.sum((retval_0[ind_in] - y_obs[mask])**2 / 2.))
except:
print 'retval_0[ind_in] and y_obs[mask] are not equal'
diff = np.append(diff, 100000)
print 'difference:', diff[-1], gamma
# if do_plot:
# plt.title('%d, %d, %d, %.4f'%(gamma, y_obs[ind_cv].shape[0], date_obs[mask].shape[0], diff))
# plt.fill_between(t, retval_0 + post_sd_0, retval_0 - post_sd_0, color="0.8")
# plt.plot(t, retval_0)
# plt.plot(t[ind_in], retval_0[ind_in], 'o', c='y')
# plt.plot(date_obs[mask], y_obs[mask], '.', c = 'r')
# plt.plot(date_obs[ind_cv], y_obs[ind_cv], '.', c='g')
# plt.show()
gamma = gammas[np.argmin(diff)]
print 'OPtimal gamma = ', gamma
else:
gamma = 1
# retval = np.zeros(t.shape) * -1
# post_sd = np.zeros(t.shape) * -1
for i in xrange(img_diff.shape[1]):
for j in xrange(img_diff.shape[2]):
ind = np.logical_and(img_err[:, i, j] != 999, jul_days[:] != 0)
if img_diff[ind, i, j] != []:
print 'gamma:', gamma
print 'ind:', len(ind)
# self.solve_lin(img[ind, i, j], img[ind, i, j], t, img_err[ind, i, j])
if np.sum(jul_days[ind]) != 0:
retval_0, post_sd_0, sigma_prior, sigma_obs_t = self.do_opt(img_diff[ind, i, j],\
img_err_air[ind, i, j],\
jul_days[ind],\
gamma=gamma,\
t=t_out)
retval = retval_0
post_sd = post_sd_0
img_out[:, i, j] = retval
img_out_sd[:, i, j] = post_sd
else:
img_out[:, i, j] = 0
img_out_sd[:, i, j] = 0
def fill_zeros(arr):
for ii in xrange(1, arr.shape[0]):
if np.sum(arr[ii, :, :]) == 0:
arr[ii, :, :] = arr[ii - 1, :, :]
if np.sum(arr[0, :, :]) == 0:
arr[0, :, :] = arr[1, :, :]
return arr
img_air = fill_zeros(img_air)
#!!!!!!!!!!!
# Let's get back to LST
img_air_mean = self.mean_img(img_air, t, t_out)
img_air_mean = self.mean_img(img_air, jd_air, t_out)
print 'img_air_mean: ', img_air_mean.shape
img_air_mean = fill_zeros(img_air_mean)
img_air_mean[img_out == 0] = 0
img_out = img_air_mean - img_out
#!!!!!!!!!!!
# Save retrieved data to netCDF. Change dimensions by np.moveaxis() for geoserver.
# self.save_netcdf(img_out, img_out_sd, img, img_err, t, jul_days, ncd_file, geo, proj)
# self.save_netcdf(np.moveaxis(img_out, 0, 2),\
# np.moveaxis(img_out_sd, 0, 2),\
# np.moveaxis(img_orig, 0, 2),\
# np.moveaxis(img_err, 0, 2),\
# jul_days, t_out, ncd_file)
self.save_netcdf(img_out, img_out_sd, img_orig, img_err, jul_days,
t_out, ncd_file)
# get indices where we have observations.
# ind = np.logical_and(img_err[:, 0, 0] != 999, jul_days[:] != 0)
ind = img_err[:, cx, cy] != 999
do_plot = False
if np.logical_and(do_plot == True, np.max(ind) != False):
# plt.fill_between(t, retval+post_sd, retval-post_sd, color="0.8")
# plt.plot(t, retval, color='Crimson')
# ind = img_orig == 0
plt.title('ulpx=%d, ulpy=%d:' % (ulpx, ulpy))
# plt.fill_between(t_out,\
# img_out[:, 0, 0] + img_out_sd[:, 0, 0], \
# img_out[:, 0, 0] - img_out_sd[:, 0, 0], color="0.8")
post_sd = img_out_sd[:, cx, cy]
ci_5 = np.sqrt(2) * ssp.erfinv(0.05)
ci_25 = np.sqrt(2) * ssp.erfinv(0.25)
ci_75 = np.sqrt(2) * ssp.erfinv(0.75)
plt.fill_between(t_out,
img_out[:, cx, cy] - ci_75 * post_sd,
img_out[:, cx, cy] + ci_75 * post_sd,
color='0.9')
plt.fill_between(t_out,
img_out[:, cx, cy] - ci_25 * post_sd,
img_out[:, cx, cy] + ci_25 * post_sd,
color='0.8')
plt.fill_between(t_out,
img_out[:, cx, cy] - ci_5 * post_sd,
img_out[:, cx, cy] + ci_5 * post_sd,
color='0.6')
plt.plot(t_out,
img_out[:, cx, cy],
color='Crimson',
label='Solution and associated uncertainties')
plt.errorbar(jul_days[ind], img_orig[ind, cx, cy], yerr = img_err[ind, cx, cy],\
ecolor='g', marker='.', ls='', label='Original LST and associated uncertainties')
# plt.errorbar(jul_days[ind], img[ind, 0, 0], yerr=img_err[ind, 0, 0], ecolor='g', marker='o', c='g', ls='')
plt.plot(jd_air,
img_air[:, cx, cy],
color='c',
marker='.',
label='Air Temperature')
plt.legend(loc=0)
plt.xlabel('Julian day')
plt.ylabel(('LST, K'))
# plt.ylim(-4.35 , -4.3)
# plt.ylim(280, 310)
plt.show()
plt.savefig('fig/therm_opt_test.png')
print img_diff.shape
def do_opt(self, obs, obs_sd, date_jul, gamma=70, t=np.arange(1, 365, 1)):
"""
Do optimization for given vector of observation obs
:param obs: vector of observations
:param obs_sd: observational uncertainties
:param date_jul: vector of julian days
:param gamma: dynamical model error (smoothness)
:param t: time grid
:return:
"""
# Define output dates.
# We want to work with whole years
year = julday.jul2date2(date_jul[0]).year
jd1 = julday.date2jul(dt.date(year, 1, 1))
year = julday.jul2date2(date_jul[-1]).year
jd2 = julday.date2jul(dt.date(year, 12, 31))
# t = np.arange(jd1, jd2 + 1, 4)
t_obs = np.arange(jd1, jd2 + 1, 1)
x_prior = np.ones_like(t) * np.mean(obs)
y_obs = np.zeros(t_obs.shape)
sigma_obs = np.zeros(t_obs.shape)
for i in xrange(t_obs.shape[0] - 1):
for j in xrange(date_jul.shape[0]):
if t_obs[i] == date_jul[j]:
y_obs[i] = obs[j]
sigma_obs[i] = obs_sd[j]
obs_mask = np.zeros(t.shape[0]).astype(bool)
obs_mask = np.where(sigma_obs > 0, True, False)
sigma_obs_t = np.zeros(t.shape)
sigma_prior = 100.
retval = np.zeros(x_prior.shape[0])
post_sd = np.zeros(x_prior.shape[0])
# Sometimes fmin_bfgs() can give "input/output error". The reasons are unknown for me. So if this happens we can
# simply return zeros what flags this pixel as a bad one.
try:
retval = scipy.optimize.fmin_bfgs(jc.cost_function_t, x_prior, fprime=jc.der_cost_function_t, \
args=(y_obs, x_prior, sigma_obs, sigma_prior, gamma, obs_mask, t, t_obs),
disp=1, retall=1)[0]
except:
print 'An error in fmin_bfgs():', sys.exc_info()[0]
return retval, post_sd, sigma_prior, sigma_obs_t
# Calculate observational uncertainty in grid of output.
# This is requared for calculation of posteriour uncertainty
for i in xrange(t.shape[0] - 1):
ind = np.logical_and(t_obs >= t[i], t_obs < t[i + 1])
k = 0
for j in xrange(t_obs[ind].shape[0]):
if y_obs[ind][j] != 0:
sigma_obs_t[i] = sigma_obs_t[i] + sigma_obs[ind][j]
k += 1
if k != 0:
sigma_obs_t[i] = sigma_obs_t[i] / k
# der_j_obs[i] = (x[i] - y[ind][j]) ** 2 / sigma_obs[ind][j] ** 2
# We assume that uncertainty of no observation is very large
sigma_obs_t[sigma_obs_t == 0] = 999
# pdb.set_trace()
# Do uncertainty
post_sd = jc.posterior_covariance(retval, sigma_prior, sigma_obs_t,
gamma)
return retval, post_sd, sigma_prior, sigma_obs_t
def get_back(self, loc, sent1_name, ulpx0, ulpy0, cols, rows):
"""
Get backscatter from files from specified directory (loc).
Files must have two bands: backscatter and uncertainty
Return ndarray arrays of backscatter and uncertainty
:param loc: directory
:param sent1_name: name pattern for names to search
:return: sent1_in: backscatter; sent1_in_sd: uncertanty;
date_jul: julian day, date
"""
#pol = 'vv'
# fs = baci_fs(site)
# We need the same size for all images in time series.
# For this we use one reference image
# All images must have the same projection and geo. units!
# if fs.site['ref_file'] == '':
# if we don't have a reference for this site use as
# a reference first file in list
# f_list = np.array([file for file in glob.glob(loc + sent1_name)])
# else:
# f_list = np.array([file for file in glob.glob(loc + fs.site['ref_file'])])
f_list = np.array([file for file in glob.glob(loc + sent1_name)])
# Get geo. coord of a reference image
gds = gdal.Open(f_list[0])
geo = gds.GetGeoTransform()
mx0 = ulpx0 * geo[1] + geo[0]
my0 = ulpy0 * geo[5] + geo[3]
sent1_in = np.zeros((f_list.shape[0], cols, rows))
# print 'sent1_in.shape:', sent1_in.shape
sent1_in_sd = np.zeros((f_list.shape[0], cols, rows))
sent1_in[:] = -99
date = []
date_jul = []
for i,fname in enumerate(f_list):
print fname
gds = gdal.Open(fname)
# get a pixel of current image by geo. coords of ref. image
geo = gds.GetGeoTransform()
ulpx = np.round((mx0 - geo[0]) / geo[1]).astype(int)
ulpy = np.round((my0 - geo[3]) / geo[5]).astype(int)
s = gds.ReadRaster(xoff=ulpx, yoff=ulpy, xsize=cols, ysize=rows,
band_list=[1, 2])
# print 'g.RasterCount', g.RasterCount
tmp = np.array(struct.unpack(gds.RasterCount * cols * rows * "f", s))
ds = np.reshape(tmp, (gds.RasterCount, cols, rows))
#if np.max(ds) == -99:
# continue
sent1_in[i, :, :] = ds[0, :, :]
sent1_in_sd[i, :, :] = ds[1, :, :]
# sent1_in[:,:,i] = gds.GetRasterBand(1).ReadAsArray()
# sent1_in_sd[:, :, i] = gds.GetRasterBand(2).ReadAsArray()
f = fname.split('/')[-1]
date = np.append(date, dt.date(int(f[:4]), int(f[4:6]), int(f[6:8])))
date_jul = np.append(date_jul, julday.date2jul(date[-1]))
# print date
# print date_jul
# print 'sent1_in', sent1_in.shape
# print 'sent1_in_sd', sent1_in_sd.shape
if date != []:
ind = np.argsort(date_jul)
date_jul = date_jul[ind]
date = date[ind]
sent1_in = sent1_in[ind,:,:]
sent1_in_sd = sent1_in_sd[ind,:,:]
#return 10 * np.log10(sent1_in), 10 * np.log10(sent1_in_sd), date_jul, date
return sent1_in, sent1_in_sd, date_jul, date
def do_job(self, ncd_out, downsample_dir, file_pattern, year, tile, ulx, uly, lrpx, lrpy, cc, rr, step=1):
print 'Starting doing main job with ' + ncd_out
site =''
#f_list = np.array([file for file in glob.glob(downsample_dir + sent1_mod_file)])
file_in_1 = file_pattern % (year, tile)
# get backscatter for the current year for block (ulx, uly) with size cc/rr (columns rows)
sent1_in, sent1_in_sd, date_jul, date = self.get_back(downsample_dir, file_in_1, ulx, uly, cc, rr)
# get first and last julian days for 'year'
year = julday.jul2date2(date_jul[0]).year
d1 = julday.date2jul(dt.date(year, 1, 1))
d2 = julday.date2jul(dt.date(year, 12, 31))
# get temporal range
self.t = np.arange(d1, d2 + 1, 1)
t_out = np.arange(d1, d2 + 1, step)
head_flag = False
tail_flag = False
# get filename for previous year
file_in_0 = file_pattern % (year - 1, tile)
f_list = np.array([file for file in glob.glob(downsample_dir + file_in_0)])
# if data exist get backscatter for previous year
if len(f_list) != 0:
sent1_in_0, sent1_in_0_sd, date_jul_0, date_0 = self.get_back(downsample_dir, file_in_0, ulx, uly, cc, rr)
# get data if we have any good pixels
if np.max(sent1_in_0[:]) != -99:
# we need last day for this year therefore looping backward
for i in xrange(sent1_in_0.shape[0]-1, 0, -1):
if np.max(sent1_in_0[i, :, :]) != -99:
tmp = np.zeros((sent1_in.shape[0]+1, sent1_in.shape[1], sent1_in.shape[2]))
tmp[1:,:, :] = sent1_in[:]
tmp[0, :, :] = sent1_in_0[i, :, :]
sent1_in = tmp.copy()
tmp[1:,:, :] = sent1_in_sd[:]
tmp[0, :, :] = sent1_in_0_sd[i,:,:]
sent1_in_sd = tmp.copy()
head_flag = True
break
# add first date
date_jul = np.append(d1, date_jul)
date = np.append(dt.date(year, 1, 1), date)
# get filename for next year
file_in_2 = file_pattern % (year + 1, tile)
f_list = np.array([file for file in glob.glob(downsample_dir + file_in_2)])
if len(f_list) != 0:
sent1_in_2, sent1_in_2_sd, date_jul_2, date_2 = self.get_back(downsample_dir, file_in_2, ulx, uly, cc, rr)
if np.max(sent1_in_2[:]) != -99:
for i in xrange(0, sent1_in_2.shape[2]):
if np.max(sent1_in_2[i, :, :]) != -99:
tmp = np.zeros((sent1_in.shape[0]+1, sent1_in.shape[1], sent1_in.shape[2]))
tmp[:-1,:, :] = sent1_in[:]
tmp[-1, :, :] = sent1_in_2[i, :, :]
sent1_in = tmp.copy()
tmp[:-1, :, :] = sent1_in_sd[:]
tmp[-1, :, :] = sent1_in_2_sd[i, :, :]
sent1_in_sd = tmp.copy()
tail_flag = True
break
# add last date
date_jul = np.append(date_jul, d2)
date = np.append(date, dt.date(year, 12, 31))
#t = np.arange(date_jul[0] - 1, date_jul[-1] + 1)
sent1_out = np.zeros((t_out.shape[0], sent1_in.shape[1], sent1_in.shape[2]))
sent1_out_sd = np.zeros((t_out.shape[0], sent1_in.shape[1], sent1_in.shape[2]))
if np.max(sent1_in) != -99:
for i in xrange(sent1_in.shape[1]):
for j in xrange(sent1_in.shape[2]):
#print 'pixel %d %d' % (i,j)
# retval, post_sd = self.do_opt2(sent1_in[i, j, :], sent1_in_sd[i, j, :], date_jul, date, t_out)
retval, post_sd, sigma_prior, sigma_obs_t = self.do_opt3(sent1_in[:, i, j],\
sent1_in_sd[:, i, j],\
date_jul, gamma=1, t=t_out)
# sent1_out[i, j, :] = 10 * np.log10(retval)
# sent1_out_sd[i, j, :] = 10 * np.log10(post_sd)
sent1_out[:, i, j] = retval
sent1_out_sd[:, i, j] = post_sd
if head_flag == True:
sent1_in = np.delete(sent1_in, 0, axis=0)
sent1_in_sd = np.delete(sent1_in_sd, 0, axis=0)
date_jul = np.delete(date_jul, 0)
date = np.delete(date, 0)
if tail_flag == True:
sent1_in = np.delete(sent1_in, -1, axis=0)
sent1_in_sd = np.delete(sent1_in_sd, -1, axis=0)
date_jul = np.delete(date_jul, -1)
date = np.delete(date, -1)
print 'sent1_in:', sent1_in.shape
# self.save_netcdf(sent1_out, sent1_out_sd, self.t, 'sentinel/sent1_%d_%d_%d_%d.nc' % (ulx, uly, cc, rr))
#ind = np.in1d(self.t, date_jul)
ind = []
for ii, d0 in enumerate(self.t):
for d1 in date_jul:
if d0 == d1:
ind = np.append(ind, ii).astype(int)
bs_orig = np.ones(sent1_in.shape) * 10000
bs_orig_sd = np.ones(sent1_in.shape) * 10000
# bs_orig[:,:,ind] = sent1_in
# bs_orig_sd[:, :, ind] = sent1_in_sd
bs_orig[:] = sent1_in
bs_orig_sd[:] = sent1_in_sd
bs_orig[np.isnan(bs_orig)] = 10000
bs_orig_sd[np.isnan(bs_orig_sd)] = 10000
# Save result to netCDF file
self.save_netcdf(bs_orig, bs_orig_sd, date_jul, sent1_out, sent1_out_sd, t_out, ncd_out)
print 'run_job series is done!!!'
do_plot = False
if do_plot:
i = 0
j = 0
# t = np.arange(date_jul[0] - 1, date_jul[-1] + 1)
ci_5 = np.sqrt(2) * ssp.erfinv(0.05)
ci_25 = np.sqrt(2) * ssp.erfinv(0.25)
ci_75 = np.sqrt(2) * ssp.erfinv(0.75)
plt.fill_between(t_out, sent1_out[:, i, j] - ci_75 * post_sd, sent1_out[:, i, j] + ci_75 * post_sd,
color='0.9')
plt.fill_between(t_out, sent1_out[:, i, j] - ci_25 * post_sd, sent1_out[:, i, j] + ci_25 * post_sd,
color='0.8')
plt.fill_between(t_out, sent1_out[:, i, j] - ci_5 * post_sd, sent1_out[:, i, j] + ci_5 * post_sd,
color='0.6')
# plt.fill_between(t, retval[0])
print i,j
plt.plot(t_out, sent1_out[:, i, j], label='Interpolated backscatter')
# plt.errorbar(date_jul, 10 * np.log10(sent1_in[i, j, :]), yerr=10 * np.log10(sent1_in_sd[i, j, :]),
# marker='o', ls='')
ind = sent1_in[:, i, j] > -90
try:
plt.errorbar(date_jul[ind], sent1_in[ind, i, j], yerr=sent1_in_sd[ind, i, j],
marker='o', ls='', label='Observations')
except:
print 'can not plot errorbar. sorry.'
#plt.ylim(-5, -15)
plt.xlabel('Julian day', fontsize='large')
plt.ylabel('Backscatter', fontsize='large')
plt.legend(loc=0, fontsize='large')
plt.show()
def do_opt2(self, back_scat, back_scat_sd, date_jul, date, t = np.arange(1, 365, 1)):
"""
Find optimal smooth solution for time series of backsctter data
taking uncertainties into account.
:param back_scat:
:param back_scat_sd:
:param date_jul:
:param date:
:return:
"""
# back_scat = bs
# back_scat = bs_sd
# date_jul = dj_opt
# date = d_opt
year = julday.jul2date2(date_jul[0]).year
d1 = julday.date2jul(dt.date(year, 1, 1))
year = julday.jul2date2(date_jul[-1]).year
d2 = julday.date2jul(dt.date(year, 12, 31))
x = np.arange(d1, d2 + 1)
ind = np.logical_and(~np.isnan(back_scat), back_scat > -90)
back_scat = back_scat[ind]
back_scat_sd = back_scat_sd[ind]
date_jul = date_jul[ind]
date = date[ind]
x_prior = np.ones_like(x) * np.mean(back_scat)
y_obs = np.zeros(x.shape)
sigma_obs = np.zeros(x.shape)
sigma_obs[:] = 99
for i in xrange(x.shape[0]):
for j in xrange(date_jul.shape[0]):
if x[i] == date_jul[j]:
y_obs[i] = back_scat[j]
sigma_obs[i] = back_scat_sd[j]
obs_mask = np.ones(x.shape[0]).astype(int)
obs_mask = np.where(y_obs != 0, True, False)
sigma_prior = 100.
gamma = 1
# Do optimization of sentinel data in order to restore gaps
# print 'starting fmin_bfgs with y_obs ', y_obs[obs_mask].shape
# plt.plot(y_obs[obs_mask])
# plt.plot(x_prior[obs_mask])
# plt.show()
# retval = scipy.optimize.fmin_bfgs(jc.cost_function, x_prior, fprime=jc.der_cost_function, \
# args=(y_obs[obs_mask], x_prior, sigma_obs[obs_mask], sigma_prior, gamma,
# obs_mask), disp=1, retall=15, maxiter=1500)
bounds = np.zeros((x_prior.shape[0], 2))
bounds[:, 0] = -200
bounds[:, 1] = 0.5
if y_obs[obs_mask] != []:
retval = scipy.optimize.fmin_l_bfgs_b(jc.cost_function, x_prior, fprime=jc.der_cost_function, \
args=(y_obs[obs_mask], x_prior, sigma_obs[obs_mask], sigma_prior, gamma,
obs_mask),\
bounds = bounds, factr=10000000, pgtol=1e-05, maxiter=1500,\
iprint=0)[0]
# disp=0,
# Do uncertainty
try:
post_sd = jc.posterior_covariance(retval, sigma_prior, sigma_obs, gamma, obs_mask)
except:
print 'jc.posterior_covariance exception'
post_sd = np.ones(x.shape[0])
else:
retval = np.ones(x.shape[0])
post_sd = np.ones(x.shape[0])
ind = []
for jj in xrange(x.shape[0]):
if x[jj] in self.t:#x0:
ind = np.append(ind, jj).astype(int)
return retval[ind], post_sd[ind]
def do_opt(self, bs, bs_sd, dj_opt, d_opt):
"""
Find optimal smooth solution for time series of backsctter data
taking uncertainties into account.
:param back_scat:
:param back_scat_sd:
:param date_jul:
:param date:
:return:
"""
# bs = np.concatenate((sent1_in[i, j, :], sent1_in[i, j, :], sent1_in[i, j, :]))
# bs_sd = np.concatenate((sent1_in_sd[i, j, :], sent1_in_sd[i, j, :], sent1_in_sd[i, j, :]))
# dj_opt = np.concatenate((date_jul_1, date_jul, date_jul_2))
# d_opt = np.concatenate((date_1, date, date_2))
# date0 = []
#x0 = np.arange(dj_opt[0] - 1, dj_opt[-1] + 1)
dj_opt1 = []
dj_opt2 = []
d_opt1 = []
d_opt2 = []
for jd in dj_opt:
d = julday.jul2date2(jd).date()
# print d
d0 = datetime.date(d.year - 1, d.month, d.day)
d_opt1 = np.append(d_opt1, d0)
# print date0[-1]
dj_opt1 = np.append(dj_opt1, julday.date2jul(d0))
d0 = datetime.date(d.year + 1, d.month, d.day)
d_opt2 = np.append(d_opt2, d0)
dj_opt2 = np.append(dj_opt2, julday.date2jul(d0))
# date_1 = np.append(date_1, datetime.date())
back_scat = np.concatenate((bs, bs, bs))
back_scat_sd = np.concatenate((bs_sd, bs_sd, bs_sd))
date_jul = np.concatenate((dj_opt1, dj_opt, dj_opt2))
date = np.concatenate((d_opt1, d_opt, d_opt2))
# back_scat = bs
# back_scat_sd = bs_sd
# date_jul = dj_opt
# date = d_opt
year = julday.jul2date2(date_jul[0]).year
d1 = julday.date2jul(dt.date(year, 1, 1))
year = julday.jul2date2(date_jul[-1]).year
d2 = julday.date2jul(dt.date(year, 12, 31))
x = np.arange(d1, d2 + 1)
#x = np.arange(date_jul[0] - 1, date_jul[-1] + 1)
ind = ~np.isnan(back_scat)
back_scat = back_scat[ind]
back_scat_sd = back_scat_sd[ind]
date_jul = date_jul[ind]
date = date[ind]
x_prior = np.ones_like(x) * np.mean(back_scat)
# print x
y_obs = np.zeros(x.shape)
sigma_obs = np.zeros(x.shape)
for i in xrange(x.shape[0]):
for j in xrange(date_jul.shape[0]):
if x[i] == date_jul[j]:
y_obs[i] = back_scat[j]
sigma_obs[i] = back_scat_sd[j]
obs_mask = np.ones(x.shape[0]).astype(int)
obs_mask = np.where(y_obs > 0, True, False)
# print 'obs_mask\n', obs_mask
# print 'y_obs\n', y_obs
sigma_prior = 100.
gamma = 1
# sigma_obs = 0.015
# os.chdir('/home/ucfamc3/max_python/baci_brdf/')
# Do optimization of sentinel data in order to restore gaps
print 'starting fmin_bfgs with y_obs ', y_obs[obs_mask].shape
retval = scipy.optimize.fmin_bfgs(jc.cost_function, x_prior, fprime=jc.der_cost_function, \
args=(y_obs[obs_mask], x_prior, sigma_obs[obs_mask], sigma_prior, gamma,
obs_mask), disp=1, retall=15, maxiter=1500)
bounds = bounds = np.zeros((x_prior.shape[0], 2))
bounds[:, 0] = 0
bounds[:, 1] = 0.5
# retval = scipy.optimize.fmin_l_bfgs_b(jc.cost_function, x_prior, fprime=jc.der_cost_function, \
# args=(y_obs[obs_mask], x_prior, sigma_obs[obs_mask], sigma_prior, gamma,
# obs_mask),\
# bounds = bounds, factr=10000000, pgtol=1e-05, maxiter=1500, disp=1)
# retval = scipy.optimize.minimize(jc.cost_function, x_prior, method="L-BFGS-B", \
# jac=True, bounds=the_bounds, options=self.optimisation_options)
# retval = scipy.optimize.minimize(jc.cost_function, x_prior, method="L-BFGS-B", \
# args=(y_obs[obs_mask], x_prior, sigma_obs[obs_mask], sigma_prior, gamma, obs_mask),\
# jac=False)
#
# self.optimisation_options = {"factr": 1000, \
# "m": 400, "pgtol": 1e-12, "maxcor": 200, \
# "maxiter": 1500, "disp": True}
# Do uncertainty
post_sd = jc.posterior_covariance(retval[0], sigma_prior, sigma_obs, gamma, obs_mask)
ind = []
for jj in xrange(x.shape[0]):
if x[jj] in self.t:#x0:
ind = np.append(ind, jj).astype(int)
return retval[0][ind], post_sd[ind]
def copy_ds_mod11(nc_big, ds, x1, x2, y1, y2, year):
"""
copy data from small datasets to one big netCDF dataset with MOD11 LST
"""
# ds.variables['lst'][:][ds.variables['lst'][:]==0] = np.ma.masked
# ds.variables['lst_sd'][:][ds.variables['lst_sd'][:] == 0] = np.ma.masked
x3 = nc_big[0].variables['lst'][:, y1:y2, x1:x2].shape[2]
y3 = nc_big[0].variables['lst'][:, y1:y2, x1:x2].shape[1]
print 'size block:', ds.variables['lst'][:, 0:y3, 0:x3].shape[0]
print 'size big:', nc_big[0].variables['lst'][:, y1:y2, x1:x2].shape[0]
# MaskedArrayFutureWarning: setting an item on a masked array which has a shared mask will not copy the mask and
# also change the original mask array in the future.
# Copy one of the blocks to output array
nc_big[0].variables['lst'][:, y1:y2, x1:x2] = np.ma.masked_values(
ds.variables['lst'][:, 0:y3, 0:x3], 0)
# nc_big[0].variables['lst'][:][y1:y2, x1:x2, :].unshare_mask()
# Mask zero-pixels (water, etc.)
# ind = nc_big[0].variables['lst'][y1:y2, x1:x2, :] == 0
# nc_big[0].variables['lst'][:][y1:y2, x1:x2, :][ind] = True #np.ma.masked
# nc_big[0].variables['lst_sd'][:][y1:y2, x1:x2, :] = ds.variables['lst_sd'][0:y3, 0:x3, :]
nc_big[0].variables['lst_sd'][:, y1:y2, x1:x2] = np.ma.masked_values(
ds.variables['lst_sd'][:, 0:y3, 0:x3], 0)
# nc_big[0].variables['lst_sd'][:][y1:y2, x1:x2, :].unshare_mask()
# ind = nc_big[0].variables['lst_sd'][:][y1:y2, x1:x2, :] == 0
# nc_big[0].variables['lst_sd'][:][y1:y2, x1:x2, :][ind] = np.ma.masked
# if np.min(ds.variables['lst'][:]) == 0:
# pdb.set_trace()
#print 'min/max:', np.min(nc_big.variables['lst'][0:y3, 0:x3, :]), np.max(ds.variables['lst'][0:y3, 0:x3, :])
#print 'min/max:', np.min(nc_big.variables['lst'][y1:y2, x1:x2, :]), np.max(ds.variables['lst'][0:y3, 0:x3, :])
#nc_big.variables['date_jul'][:] = ds.variables['date_jul'][:]
jd1 = julday.date2jul(dt.date(year, 1, 1))
jd = np.arange(jd1, jd1 + ds.variables['lst'].shape[0])
nc_big[0].variables['julday'][:] = ds.variables['date_jul'][:] #jd
#nc_big.variables['lat'][y1:y2, x1:x2] = ds.variables['lat'][:].T
#nc_big.variables['lon'][y1:y2, x1:x2] = ds.variables['lon'][:].T
#********
# original dates are always different
#julday.jul2doy()
# pdb.set_trace()
# mask = np.array([julday.jul2doy(d) for d in ds.variables['date_jul_obs'][:]]) - 1
#print mask
# print ds.variables['date_jul_obs'][:].shape
#print ds.variables['lst_orig'][0:y3, 0:x3, :]
#print mask.shape
#print nc_big.variables['lst_orig'][y1:y2, x1:x2, :].shape
orig_size = ds.variables['lst_orig'][:, 0:y3, 0:x3].shape[0]
# print 'nc_big[1].variables[lst_orig][:orig_size, y1:y2, x1:x2]: ', nc_big[1].variables['lst_orig'][:orig_size, y1:y2, x1:x2].shape
# print 'ds.variables[lst_orig][:, 0:y3, 0:x3]: ', ds.variables['lst_orig'][:, 0:y3, 0:x3].shape
nc_big[1].variables['lst_orig'][:orig_size, y1:y2, x1:x2] =\
np.ma.masked_values(ds.variables['lst_orig'][:, 0:y3, 0:x3], 0)
nc_big[1].variables['lst_orig_sd'][:orig_size, y1:y2, x1:x2] = \
np.ma.masked_values(ds.variables['lst_orig_sd'][:, 0:y3, 0:x3], 0)
nc_big[1].variables['julday_obs'][:orig_size] = ds.variables[
'date_jul_obs'][:]