LabelMap, PolarMap = MapDivide(GCZ_map, 35)
mapUS = LabelMap[194:722, 984:2320] #Clip the region of interest
labels = np.unique(mapUS)[1:]
US_labels = [int(x) for x in labels]
for jj in range(14, 20):
Li = (LabelMap == jj).astype(np.uint8)
rows, cols = np.where(Li == 1)
[rl, ru, cl, cr] = [np.min(rows), np.max(rows), np.min(cols), np.max(cols)]
print jj, [rl, ru, cl, cr]
r17 = np.load(
'/ssd-scratch/htranvie/Snow_dataset/filtered_multiparts/oyscacldf4_17.0.npy'
)
t0 = date(2000, 2, 24)
t_check = date(2003, 7, 6)
t_start = t_check - timedelta(days=15)
t_end = t_check + timedelta(days=14)
ndays = (t_end - t_start).days / 2 + 1
dn_start1 = (t_start - t0).days
dn_end1 = (t_end - t0).days
oyscacldf4_17 = r17[dn_start1:dn_end1, :, :]
oyscacldf4_18 = r18[dn_start1:dn_end1, :, :]
oyscacldf4_17i = ClearCloud(oyscacldf4_17, t_start, t_end)
oyscacldf4_18i = ClearCloud(oyscacldf4_18, t_start, t_end)
oyscacldf4 = np.vstack([oyscacldf4_17i, oyscacldf4_18i])
oyscacldf4 = oyscacldf4[:, 984 - 897:]
fvi = gdal.Open('result_US/SnowUS_' + t_check.strftime('%Y-%m-%d') +
'.asc').ReadAsArray()
oyscacldf4_fvi = fvi[216 - 194:657 - 194 + 1, :1546 - 984 + 1]
else:
arr0 = gdal.Open(bin_files[0]).ReadAsArray()
oyflood = np.concatenate([oyflood,np.zeros((1,arr_temp.shape[0],arr_temp.shape[1]))],axis=0)
oyflood = np.concatenate([oyflood,arr0.reshape(-1,arr_temp.shape[0],arr_temp.shape[1])],axis=0)
else:
for bin_file in bin_files:
arr1 = gdal.Open(bin_file).ReadAsArray()
oyflood = np.concatenate([oyflood,arr1.reshape(-1,arr_temp.shape[0],arr_temp.shape[1])],axis=0)
oyflood[oyflood==1]=2
oyflood[oyflood==-1]=1
scipy.io.savemat('/ssd-scratch/htranvie/Flood/data/oyflood',mdict={'oyflood':oyflood})
oyscacld = scipy.io.loadmat('/ssd-scratch/htranvie/Flood/data/oyflood.mat')['oyflood']
oyscacld1 = ndimage.zoom(oyscacld,(1,4,4),order=0)
sca = ClearCloud(oyscacld1,date(2008,6,5),date(2008,6,5)+timedelta(days=29))
scipy.io.savemat('/ssd-scratch/htranvie/Flood/data/sca_py3',mdict={'sca_py3':sca})
sca = scipy.io.loadmat('/ssd-scratch/htranvie/Flood/data/sca_py2.mat')['sca_py2']
#for matlab
sca = scipy.io.loadmat('/ssd-scratch/htranvie/Flood/data/sca2.mat')['sca2']
sca = np.swapaxes(np.swapaxes(sca,0,2),1,2)
#Draw report
t0 = date(2008,6,5)
fig=plt.figure()
m1 = 4
n = 6
for i in range(n*m1,n*m1+n):
t = t0+timedelta(days=i/2)
ax=fig.add_subplot(2,n,i-n*m1+1)
def BootstrapTest(oyscacld, ntimes):
cld = scipy.io.loadmat('cloud_lib.mat')['cld']
cldpct = scipy.io.loadmat('cloud_lib.mat')['cldpct']
[nday, nr, nc] = oyscacld.shape
pod = np.array([]).reshape(0, 4).astype(np.float32)
far = np.array([]).reshape(0, 4).astype(np.float32)
csi = np.array([]).reshape(0, 4).astype(np.float32)
for jk in range(ntimes):
lowcldday = np.zeros((nday, nr, nc)).astype(np.uint8)
medcldday = np.zeros((nday, nr, nc)).astype(np.uint8)
highcldday = np.zeros((nday, nr, nc)).astype(np.uint8)
sevcldday = np.zeros((nday, nr, nc)).astype(np.uint8)
for i in range(nday):
##Low cloud hindrance
pctls = np.where((cldpct > 0.03) & (cldpct <= 0.05))[1]
iii = np.random.randint(len(pctls), size=1)[0]
lowcld = cld[:, :, pctls[iii]]
lowcld1 = oyscacld[i, :, :] * 2 + lowcld
lowcld1[lowcld1 == 3] = 1
lowcldday[i, :, :] = lowcld1
##Med cloud hindrance
pctls = np.where((cldpct > 0.07) & (cldpct <= 0.09))[1]
iii = np.random.randint(len(pctls), size=1)[0]
medcld = cld[:, :, pctls[iii]]
medcld1 = oyscacld[i, :, :] * 2 + medcld
medcld1[medcld1 == 3] = 1
medcldday[i, :, :] = medcld1
##High cloud hindrance
pctls = np.where((cldpct > 0.15) & (cldpct <= 0.2))[1]
iii = np.random.randint(len(pctls), size=1)[0]
highcld = cld[:, :, pctls[iii]]
highcld1 = oyscacld[i, :, :] * 2 + highcld
highcld1[highcld1 == 3] = 1
highcldday[i, :, :] = highcld1
##Severe cloud hindrance
pctls = np.where((cldpct > 0.2) & (cldpct <= 0.4))[1]
iii = np.random.randint(len(pctls), size=1)[0]
sevcld = cld[:, :, pctls[iii]]
sevcld1 = oyscacld[i, :, :] * 2 + sevcld
sevcld1[sevcld1 == 3] = 1
sevcldday[i, :, :] = sevcld1
sca1 = ClearCloud(lowcldday, date(2009, 9, 9), date(2009, 9, 10),
date(2009, 9, 15))
sca2 = ClearCloud(medcldday, date(2009, 9, 9), date(2009, 9, 10),
date(2009, 9, 15))
sca3 = ClearCloud(highcldday, date(2009, 9, 9), date(2009, 9, 10),
date(2009, 9, 15))
sca4 = ClearCloud(sevcldday, date(2009, 9, 9), date(2009, 9, 10),
date(2009, 9, 15))
lowhit = np.sum(
np.logical_and(oyscacld[2, :, :] == 1, sca1[2, :, :] == 1))
lowmiss = np.sum(
np.logical_and(oyscacld[2, :, :] == 1, sca1[2, :, :] == 0))
lowfalse = np.sum(
np.logical_and(oyscacld[2, :, :] == 0, sca1[2, :, :] == 1))
medhit = np.sum(
np.logical_and(oyscacld[2, :, :] == 1, sca2[2, :, :] == 1))
medmiss = np.sum(
np.logical_and(oyscacld[2, :, :] == 1, sca2[2, :, :] == 0))
medfalse = np.sum(
np.logical_and(oyscacld[2, :, :] == 0, sca2[2, :, :] == 1))
highhit = np.sum(
np.logical_and(oyscacld[2, :, :] == 1, sca3[2, :, :] == 1))
highmiss = np.sum(
np.logical_and(oyscacld[2, :, :] == 1, sca3[2, :, :] == 0))
highfalse = np.sum(
np.logical_and(oyscacld[2, :, :] == 0, sca3[2, :, :] == 1))
sevhit = np.sum(
np.logical_and(oyscacld[2, :, :] == 1, sca4[2, :, :] == 1))
sevmiss = np.sum(
np.logical_and(oyscacld[2, :, :] == 1, sca4[2, :, :] == 0))
sevfalse = np.sum(
np.logical_and(oyscacld[2, :, :] == 0, sca4[2, :, :] == 1))
pod = np.vstack([
pod,
np.array([
lowhit / float(lowhit + lowmiss),
medhit / float(medhit + medmiss),
highhit / float(highhit + highmiss),
sevhit / float(sevhit + sevmiss)
])
])
far = np.vstack([
far,
np.array([
lowfalse / float(lowhit + lowfalse),
medfalse / float(medhit + medfalse),
highfalse / float(highhit + highfalse),
sevfalse / float(sevhit + sevfalse)
])
])
csi = np.vstack([
csi,
np.array([
lowhit / float(lowhit + lowmiss + lowfalse),
medhit / float(medhit + medmiss + medfalse),
highhit / float(highhit + highmiss + highfalse),
sevhit / float(sevhit + sevmiss + sevfalse)
])
])
return pod, far, csi
import scipy.ndimage
d0 = date(2013, 6, 7)
for i in [0, 48]:
#for i in [377,409,425,441]:
d1 = d0 + timedelta(days=i)
# if os.path.isfile('Cloud_free/MOD_CF_'+d1.strftime('%Y%m%d')+'.npy'):
# continue
selectedDays = [(d1+timedelta(days=-x-1)).strftime('%Y%m%d') for x in range(7)][::-1]+\
[(d1+timedelta(days=x)).strftime('%Y%m%d') for x in range(8)]
refl_in = np.array([]).reshape(0, 1494, 1836)
count = 0
for j, day in enumerate(selectedDays):
if os.path.isfile('results/MOD_' + day + '_bin.tif'):
arr = io.imread('results/MOD_' + day + '_bin.tif')
else:
count += 1
arr = np.ones((1494, 1836)) * -1
refl_in = np.vstack([arr[np.newaxis, :, :], refl_in])
if count > 7:
continue
refl_in[refl_in == 1] = 2
refl_in[refl_in == -1] = 1
#L = scipy.ndimage.zoom(refl_in,(1,2,2),order=1)
refl_vi = ClearCloud(refl_in, d1 + timedelta(days=-7),
d1 + timedelta(days=7))
plt.clf()
plt.imshow(np.ma.masked_where(refl_vi == -99, refl_vi))
plt.savefig('Cloud_free/img/h' + d1.strftime('%Y%m%d') + '.png')
np.save('Cloud_free/MOD_CF_' + d1.strftime('%Y%m%d'), refl_vi)
def CloudRemoval(yyyy1, mm1, dd1, labeli):
t0 = date(2000, 2, 24)
t_start = date(yyyy1, mm1, dd1)
t_end = t_start + timedelta(days=29)
ndays = (t_end - t_start).days / 2 + 1
if not os.path.isdir('/ssd-scratch/htranvie/Snow_dataset/results/' +
str(labeli)):
os.mkdir('/ssd-scratch/htranvie/Snow_dataset/results/' + str(labeli))
print(t_start + timedelta(days=ndays)).strftime('%Y-%m-%d')
if os.path.isfile('/ssd-scratch/htranvie/Snow_dataset/results/' +
str(labeli) + '/result_' + str(labeli) + '_' +
(t_start + timedelta(days=ndays)).strftime('%Y-%m-%d') +
'.npy'):
return
GCZ_map, classes = GetClimateZone()
LabelMap, PolarMap = MapDivide(GCZ_map, 35)
labels = np.unique(LabelMap)[1:]
Li = (LabelMap == labeli).astype(np.uint8)
rows, cols = np.where(Li == 1)
[rl, ru, cl, cu] = [np.min(rows), np.max(rows), np.min(cols), np.max(cols)]
Li_mini = Li[rl:ru + 1, cl:cu + 1]
Polar_mini = PolarMap[rl:ru + 1, cl:cu + 1]
dn_start1 = (t_start - t0).days
dn_end1 = (t_end - t0).days
oyscacldf4i = np.load(
'/ssd-scratch/htranvie/Snow_dataset/merged_multiparts/oyscacldf4_' +
str(labeli) + '.npy')[dn_start1:dn_end1, :, :]
Polar_add = np.logical_and(PolarMap == 1,
LabelMap == 0)[rl:ru + 1,
cl:cu + 1].astype(np.uint8)
start_tot = time.time()
start = time.time()
oyscacldfi = ClearCloud(oyscacldf4i, t_start, t_end)
if oyscacldfi == 'need to divide array':
oyscacldfi_1 = ClearCloud(oyscacldf4i[:, :, :(cu + 1 - cl) / 2],
t_start, t_end)
if oyscacldfi_1 == 'need to divide array':
oyscacldfi_10 = ClearCloud(oyscacldf4i[:, :, :(cu + 1 - cl) / 4],
t_start, t_end)
if oyscacldfi_10 == 'need to divide array':
oyscacldfi_100 = ClearCloud(
oyscacldf4i[:, :(ru + 1 - rl) / 2, :(cu + 1 - cl) / 4],
t_start, t_end)
oyscacldfi_101 = ClearCloud(
oyscacldf4i[:, (ru + 1 - rl) / 2:, :(cu + 1 - cl) / 4],
t_start, t_end)
oyscacldfi_10 = np.vstack([oyscacldfi_100, oyscacldfi_101])
oyscacldfi_11 = ClearCloud(
oyscacldf4i[:, :, (cu + 1 - cl) / 4:(cu + 1 - cl) / 2],
t_start, t_end)
if oyscacldfi_11 == 'need to divide array':
oyscacldfi_110 = ClearCloud(
oyscacldf4i[:, :(ru + 1 - rl) / 2,
(cu + 1 - cl) / 4:(cu + 1 - cl) / 2], t_start,
t_end)
oyscacldfi_111 = ClearCloud(
oyscacldf4i[:, (ru + 1 - rl) / 2:,
(cu + 1 - cl) / 4:(cu + 1 - cl) / 2], t_start,
t_end)
oyscacldfi_11 = np.vstack([oyscacldfi_110, oyscacldfi_111])
oyscacldfi_1 = np.hstack([oyscacldfi_10, oyscacldfi_11])
oyscacldfi_2 = ClearCloud(oyscacldf4i[:, :, (cu + 1 - cl) / 2:],
t_start, t_end)
if oyscacldfi_2 == 'need to divide array':
oyscacldfi_20 = ClearCloud(
oyscacldf4i[:, :, (cu + 1 - cl) / 2:3 * (cu + 1 - cl) / 4],
t_start, t_end)
if oyscacldfi_20 == 'need to divide array':
oyscacldfi_200 = ClearCloud(
oyscacldf4i[:, :(ru + 1 - rl) / 2,
(cu + 1 - cl) / 2:3 * (cu + 1 - cl) / 4],
t_start, t_end)
oyscacldfi_201 = ClearCloud(
oyscacldf4i[:, (ru + 1 - rl) / 2:,
(cu + 1 - cl) / 2:3 * (cu + 1 - cl) / 4],
t_start, t_end)
oyscacldfi_20 = np.vstack([oyscacldfi_200, oyscacldfi_201])
oyscacldfi_21 = ClearCloud(
oyscacldf4i[:, :, 3 * (cu + 1 - cl) / 4:], t_start, t_end)
if oyscacldfi_21 == 'need to divide array':
oyscacldfi_210 = ClearCloud(
oyscacldf4i[:, :(ru + 1 - rl) / 2,
(cu + 1 - cl) / 2:3 * (cu + 1 - cl) / 4:],
t_start, t_end)
oyscacldfi_211 = ClearCloud(
oyscacldf4i[:, (ru + 1 - rl) / 2:, 3 * (cu + 1 - cl) / 4:],
t_start, t_end)
oyscacldfi_21 = np.vstack([oyscacldfi_210, oyscacldfi_211])
oyscacldfi_2 = np.hstack([oyscacldfi_20, oyscacldfi_21])
oyscacldfi = np.hstack([oyscacldfi_1, oyscacldfi_2])
V = np.logical_and(oyscacldfi == 0, Polar_mini == 1)
oyscacldfi[V] = 1
end6 = time.time()
print 'VI process ' + str(end6 - start) + ' s'
oyscacldfi = oyscacldfi + Polar_add
end_tot = time.time()
print 'Total time ' + str(end_tot - start_tot) + ' s'
np.save(
'/ssd-scratch/htranvie/Snow_dataset/merged_multiparts/26/result_' +
str(labeli) + '_' +
(t_start + timedelta(days=ndays)).strftime('%Y-%m-%d'), oyscacldfi)
#dem_arr = io.imread('elevation/dem_new.tif')
oyscacld = np.array([]).reshape(0, 2139, 1499)
miss = 0
count = 0
for i in range(ndays):
for pre in ['MOD', 'MYD']:
t = (t_start + timedelta(days=i)).strftime('%Y%m%d')
res_files = glob('results/' + pre + '_' + t + '_bindem.tif')
if not res_files:
miss += 1
continue
elif len(res_files) == 1:
res_file = res_files[0]
else:
res_file = [x for x in res_files if 'new' in x][0]
mod_arr = gdal.Open(res_file).ReadAsArray()
mod_arr[mod_arr == 1] = 2
mod_arr[mod_arr == -1] = 1
mod_arr[river_area == -99] = -99
oyscacld = np.vstack(
[oyscacld, mod_arr[np.newaxis, 1272:, 1900:3399]])
count += 1
#L = scipy.ndimage.zoom(oyscacld,(1,2,2),order=0)
#high_lakes = np.logical_and(oyscacld==2,dem_arr>=300)
#oyscacld[high_lakes] = 0
refl_vi = ClearCloud(oyscacld)
#refl_vi[high_lakes[ndays,:,:]] = 1
scipy.io.savemat('Cloud_free/refl_vi' + t_mid.strftime('%Y%m%d') + '.mat',
mdict={'refl_vi' + t_mid.strftime('%Y%m%d'): refl_vi})
def CloudRemoval(yyyy0, mm0, dd0, yyyy1, mm1, dd1, yyyy2, mm2, dd2):
t0 = date(yyyy0, mm0, dd0)
t_start = date(yyyy1, mm1, dd1)
t_end = date(yyyy2, mm2, dd2)
ndays = (t_end - t_start).days / 2 + 1
dn_start1 = (t_start - t0).days
dn_end1 = (t_end - t0).days + 1
GCZ_map, classes = GetClimateZone()
LabelMap, PolarMap = MapDivide(GCZ_map, 35)
labels = np.unique(LabelMap)[1:]
oyscacldf4 = np.load('data/after_filter_global101516031517.npy')[
dn_start1:dn_end1, :, :]
oyscacldf1 = np.load('data/merged_global101516031517.npy')[
dn_start1:dn_end1, :, :]
n, nr, nc = oyscacldf1.shape
arr1 = np.zeros((nr, nc))
Polar_add = np.logical_and(PolarMap == 1, LabelMap == 0).astype(np.uint8)
start_tot = time.time()
for i in labels:
print str(i)
Li = (LabelMap == i).astype(np.uint8)
rows, cols = np.where(Li == 1)
[rl, ru, cl,
cu] = [np.min(rows),
np.max(rows),
np.min(cols),
np.max(cols)]
Li_mini = Li[rl:ru + 1, cl:cu + 1]
Polar_mini = PolarMap[rl:ru + 1, cl:cu + 1]
oyscacldf4i = oyscacldf4[:, rl:ru + 1, cl:cu + 1]
start = time.time()
oyscacldfi = ClearCloud(oyscacldf4i, t_start, t_end)
if oyscacldfi == 'need to divide array':
oyscacldfi_1 = ClearCloud(oyscacldf4i[:, :, :(cu + 1 - cl) / 2],
t_start, t_end)
if oyscacldfi_1 == 'need to divide array':
oyscacldfi_10 = ClearCloud(
oyscacldf4i[:, :, :(cu + 1 - cl) / 4], t_start, t_end)
if oyscacldfi_10 == 'need to divide array':
oyscacldfi_100 = ClearCloud(
oyscacldf4i[:, :(ru + 1 - rl) / 2, :(cu + 1 - cl) / 4],
t_start, t_end)
oyscacldfi_101 = ClearCloud(
oyscacldf4i[:, (ru + 1 - rl) / 2:, :(cu + 1 - cl) / 4],
t_start, t_end)
oyscacldfi_10 = np.vstack([oyscacldfi_100, oyscacldfi_101])
oyscacldfi_11 = ClearCloud(
oyscacldf4i[:, :, (cu + 1 - cl) / 4:(cu + 1 - cl) / 2],
t_start, t_end)
if oyscacldfi_11 == 'need to divide array':
oyscacldfi_110 = ClearCloud(
oyscacldf4i[:, :(ru + 1 - rl) / 2,
(cu + 1 - cl) / 4:(cu + 1 - cl) / 2],
t_start, t_end)
oyscacldfi_111 = ClearCloud(
oyscacldf4i[:, (ru + 1 - rl) / 2:,
(cu + 1 - cl) / 4:(cu + 1 - cl) / 2],
t_start, t_end)
oyscacldfi_11 = np.vstack([oyscacldfi_110, oyscacldfi_111])
oyscacldfi_1 = np.hstack([oyscacldfi_10, oyscacldfi_11])
oyscacldfi_2 = ClearCloud(oyscacldf4i[:, :, (cu + 1 - cl) / 2:],
t_start, t_end)
if oyscacldfi_2 == 'need to divide array':
oyscacldfi_20 = ClearCloud(
oyscacldf4i[:, :, (cu + 1 - cl) / 2:3 * (cu + 1 - cl) / 4],
t_start, t_end)
if oyscacldfi_20 == 'need to divide array':
oyscacldfi_200 = ClearCloud(
oyscacldf4i[:, :(ru + 1 - rl) / 2,
(cu + 1 - cl) / 2:3 * (cu + 1 - cl) / 4],
t_start, t_end)
oyscacldfi_201 = ClearCloud(
oyscacldf4i[:, (ru + 1 - rl) / 2:,
(cu + 1 - cl) / 2:3 * (cu + 1 - cl) / 4],
t_start, t_end)
oyscacldfi_20 = np.vstack([oyscacldfi_200, oyscacldfi_201])
oyscacldfi_21 = ClearCloud(
oyscacldf4i[:, :, 3 * (cu + 1 - cl) / 4:], t_start, t_end)
if oyscacldfi_21 == 'need to divide array':
oyscacldfi_210 = ClearCloud(
oyscacldf4i[:, :(ru + 1 - rl) / 2,
(cu + 1 - cl) / 2:3 * (cu + 1 - cl) / 4:],
t_start, t_end)
oyscacldfi_211 = ClearCloud(
oyscacldf4i[:, (ru + 1 - rl) / 2:,
3 * (cu + 1 - cl) / 4:], t_start, t_end)
oyscacldfi_21 = np.vstack([oyscacldfi_210, oyscacldfi_211])
oyscacldfi_2 = np.hstack([oyscacldfi_20, oyscacldfi_21])
oyscacldfi = np.hstack([oyscacldfi_1, oyscacldfi_2])
V = np.logical_and(
np.logical_and(oyscacldf1[ndays, rl:ru + 1, cl:cu + 1] == 1,
oyscacldfi == 0), Polar_mini == 1)
oyscacldfi[V] = 1
end6 = time.time()
print 'VI process ' + str(end6 - start) + ' s'
arr1[Li == 1] = oyscacldfi[Li_mini == 1]
f = plt.figure()
plt.imshow(arr1)
f.savefig('h' +
(t_start + timedelta(days=ndays)).strftime('%Y-%m-%d') +
'.png')
arr_tot = arr1 + Polar_add
arr_tot = arr_tot.astype(np.uint8)
end_tot = time.time()
print 'Total time ' + str(end_tot - start_tot) + ' s'
plotWhole(
arr_tot,
'final_' + (t_start + timedelta(days=ndays)).strftime('%Y-%m-%d'))
np.save(
'data/result_' +
(t_start + timedelta(days=ndays)).strftime('%Y-%m-%d'), arr_tot)