levels=256)
contrast.values[ii, jj] = tex.greycoprops(
glcm, 'contrast')[0].mean()
dissimilarity.values[ii, jj] = tex.greycoprops(
glcm, 'dissimilarity')[0].mean()
homogeneity.values[ii, jj] = tex.greycoprops(
glcm, 'homogeneity')[0].mean()
correlation.values[ii, jj] = tex.greycoprops(
glcm, 'correlation')[0].mean()
asm.values[ii, jj] = tex.greycoprops(glcm, 'ASM')[0].mean()
# write array to new geotiff
prefix = sfile.split('/')[-1][:-8]
res_str = str(resolution).zfill(3)
io.write_xarray_to_GeoTiff(
mean, '%s/alos_%sm/%s_%sm_mean.tif' %
(path2textures, res_str, prefix, res_str))
io.write_xarray_to_GeoTiff(
variance, '%s/alos_%sm/%s_%sm_variance.tif' %
(path2textures, res_str, prefix, res_str))
io.write_xarray_to_GeoTiff(
contrast, '%s/alos_%sm/%s_%sm_contrast.tif' %
(path2textures, res_str, prefix, res_str))
io.write_xarray_to_GeoTiff(
dissimilarity, '%s/alos_%sm/%s_%sm_dissimilarity.tif' %
(path2textures, res_str, prefix, res_str))
io.write_xarray_to_GeoTiff(
homogeneity, '%s/alos_%sm/%s_%sm_homogeneity.tif' %
(path2textures, res_str, prefix, res_str))
io.write_xarray_to_GeoTiff(
correlation, '%s/alos_%sm/%s_%sm_correlation.tif' %
CEmap = acc_stats_rf_tol0['commission_error_map']
omission_classes,omission_error_count = np.unique(lc.values[OEmap==1],return_counts=True)
commission_classes,commission_error_count = np.unique(lc_rf.values[CEmap==1],return_counts=True)
OErate=omission_error_count/class_count
CErate=commission_error_count/class_count
print('0 pixel tolerance')
print('labels',lc_labels)
print('OErate',OErate*100)
print('CErate',CErate*100)
print('PA ',(1-OErate)*100)
print('UA ',(1-CErate)*100)
print('OA ',(acc_stats_rf_tol0['OA'])*100)
# Error map
error_map = lc.copy(deep=True)
error_map.values*=np.nan
error_map.values[(acc_stats_rf_simple_tol0['omission_error_map']==0)*(lc_simple.values==1)] = 6 # no tree
error_map.values[(acc_stats_rf_simple_tol0['omission_error_map']==0)*(lc_simple.values==2)] = 5 # tree
error_map.values[(acc_stats_rf_simple_tol0['omission_error_map']==1)*
(acc_stats_rf_simple_tol1['commission_error_map']==0)*(lc_simple.values==2)] = 4 # adjacent omission error
error_map.values[(acc_stats_rf_simple_tol0['commission_error_map']==1)*
(acc_stats_rf_simple_tol1['commission_error_map']==0)*(lc_rf_simple.values==2)] = 2 # adjacent commision error
error_map.values[(acc_stats_rf_simple_tol1['omission_error_map']==1)*(lc_simple.values==2)] = 3 # omission error
error_map.values[(acc_stats_rf_simple_tol1['commission_error_map']==1)*(lc_rf_simple.values==2)] = 1 # commission error
"""
save layers to file
"""
io.write_xarray_to_GeoTiff(lc_rf,'%s%s_lc_class_rf_s1___%.0fm' % (path2output,site,dx))
io.write_xarray_to_GeoTiff(error_map,'%s%s_error_map_rf_s1___%.0fm' % (path2output,site,dx))
# add a colorbar
cax = fig.add_axes([0.33,0.15,0.33,0.025])
plt.colorbar(im,orientation='horizontal',label='AGB / Mg ha$^{-1}$',cax=cax,
extend='max')
fig.savefig('%slidar_AGB_models_and_CI95_%s.png' % (path2fig,version))
fig.show()
# write arrays to file
labels = ['median','95l','95u']
for ii, layer in enumerate([agb_median,agb95_l,agb95_u]):
outfile_20m = '%s/020m/%s_%s_%s' % (pt3_output_dir,pt3_outfile_prefix,version,labels[ii])
outfile_50m = '%s/050m/%s_%s_%s' % (pt3_output_dir,pt3_outfile_prefix,version,labels[ii])
outfile_100m = '%s/100m/%s_%s_%s' % (pt3_output_dir,pt3_outfile_prefix,version,labels[ii])
io.write_xarray_to_GeoTiff(layer,'%s' % outfile_20m)
os.system("gdalwarp -overwrite -dstnodata -9999 -tr 50 -50 -r average \
-t_srs '+proj=utm +zone=16 +datum=WGS84 +units=m +no_defs' \
%s.tif %s.tif" % (outfile_20m,outfile_50m))
os.system("gdalwarp -overwrite -dstnodata -9999 -tr 100 -100 -r average \
-t_srs '+proj=utm +zone=16 +datum=WGS84 +units=m +no_defs' \
%s.tif %s.tif" % (outfile_20m,outfile_100m))
"""
---------------------------------------------------------------------------------
REGRID TO DESIRED RESOLUTION (0.25 ha and 1 ha)
--------------------------------------------------------------------------------
"""
from scipy import ndimage as nd
import glob as glob
import sys
sys.path.append('../data_io')
import data_io as io
"""
List some file paths
"""
path2files = '/exports/csce/datastore/geos/groups/gcel/YucatanBiomass/uoe_lidar_sentinel/agb_Lidar_maps/gliht/'
path2output = '/home/dmilodow/DataStore_GCEL/YucatanBiomass/uoe_lidar_sentinel/agb_Lidar_maps/buffered_gliht/'
"""
Loop through the data files, and apply buffer in each case
"""
buffer_width = 9
ulim = 500
files = glob.glob('%s*.tif' % path2files)
count = 0
for f in files:
count += 1
print('\r%i/%i' % (count, len(files)))
ds = xr.open_rasterio(f)
mask = nd.maximum_filter(ds.values[0] < 0,
buffer_width,
mode='constant',
cval=0)
ds_new = ds.copy()
ds_new.values[0][mask] = -9999
ds_new.values[ds.values > ulim] = -9999
io.write_xarray_to_GeoTiff(ds_new.sel(band=1),
'%s%s' % (path2output, f.split('/')[-1]))
d20_bands_supres = DSen2(d10_bands,d20_bands)
# Combine the 10 m resolution layers
d10_bands = np.concatenate((d10_bands,d20_bands_supres),axis=1)
d10_bands = np.swapaxes(d10_bands,1,-1)
d10_bands = np.swapaxes(d10_bands,1,2)
# calculate the extra layers
%run /home/dmilodow/DataStore_DTM/STFC/wri_restoration_mapper/restoration-mapper/src/preprocessing/indices.py
d10_bands = evi(d10_bands, True)
d10_bands = bi(d10_bands, True)
d10_bands = msavi2(d10_bands, True)
d10_bands = si(d10_bands, True)
# load cloud mask
# !fmask_sentinel2Stacked.py -o cloud.img --safedir ../DATA/Sentinel2/awsdata/T30VUJ/L1C/S2B_MSIL1C_20190715T114359_N0208_R123_T30VUJ_20190715T150908.SAFE/
clouds = xr.open_rasterio('cloud.img')[0].sel(x=slice(W,E),y=slice(N,S)).values
cloud_and_shadow = clouds==2
cloud_and_shadow[clouds==3]=True
cloud_and_shadow[clouds==0]=True
cloud_and_shadow = resize(cloud_and_shadow,d10_bands[0,:,:,0].shape).astype('bool')
for tstep in range(0,d10_bands.shape[0]):
for band in range(0,d10_bands.shape[-1]):
d10_bands[tstep,:,:,band][cloud_and_shadow] = np.nan
# write layers to file for ingestion into next phase of the analysis.
coords = {'y':B04_10m.coords['y'].values,'x':B04_10m.coords['x'].values,'band':np.arange(d10_bands.shape[-1])+1}
outarray=xr.DataArray(d10_bands[0],coords=coords,dims=('y','x','band'))
outfile = '../DATA/Sentinel2/processed_bands_and_derivatives/%s_sentinel2_bands_10m' % site
io.write_xarray_to_GeoTiff(outarray,outfile,EPSG_CODE = '32630')
diff_std_winter = vh_mean.copy(deep=True)
vh_mean_winter.values = rescale(
np.nanmean(vh_stack[winter_mask], axis=0))
vh_std_winter.values = np.nanstd(vh_stack[winter_mask], axis=0)
vv_mean_winter.values = rescale(
np.nanmean(vv_stack[winter_mask], axis=0))
vv_std_winter.values = np.nanstd(vv_stack[winter_mask], axis=0)
diff_mean_winter.values = rescale(
np.nanmean(diff_stack[winter_mask], axis=0))
diff_std_winter.values = np.nanstd(diff_stack[winter_mask], axis=0)
print('\t Writing to geotiff')
vv_mean_file = '%s/S1A__IW__%s_%i_VV_tnr_bnr_Orb_Cal_TF_TC_dB_temporal_mean_summer.tif' % (
outdir, orbit, year)
io.write_xarray_to_GeoTiff(vv_mean_summer[0], vv_mean_file)
vh_mean_file = '%s/S1A__IW__%s_%i_VH_tnr_bnr_Orb_Cal_TF_TC_dB_temporal_mean_summer.tif' % (
outdir, orbit, year)
io.write_xarray_to_GeoTiff(vh_mean_summer[0], vh_mean_file)
diff_mean_file = '%s/S1A__IW__%s_%i_diffVVVH_tnr_bnr_Orb_Cal_TF_TC_dB_temporal_mean_summer.tif' % (
outdir, orbit, year)
io.write_xarray_to_GeoTiff(diff_mean_summer[0], diff_mean_file)
vv_std_file = '%s/S1A__IW__%s_%i_VV_tnr_bnr_Orb_Cal_TF_TC_dB_temporal_stdev_summer.tif' % (
outdir, orbit, year)
io.write_xarray_to_GeoTiff(vv_std_summer[0], vv_std_file)
vh_std_file = '%s/S1A__IW__%s_%i_VH_tnr_bnr_Orb_Cal_TF_TC_dB_temporal_stdev_summer.tif' % (
outdir, orbit, year)
io.write_xarray_to_GeoTiff(vh_std_summer[0], vh_std_file)
diff_std_file = '%s/S1A__IW__%s_%i_diffVVVH_tnr_bnr_Orb_Cal_TF_TC_dB_temporal_stdev_summer.tif' % (
outdir, orbit, year)
io.write_xarray_to_GeoTiff(diff_std_summer[0], diff_std_file)
lm = LinearRegression()
lm.fit(X, y)
print("calibration score: %.02f" % lm.score(X, y))
full2 = xr.open_rasterio('%s%s2_band%i.tif' %
(path2full, site_id, band))[0]
mask = np.isfinite(full2.values)
X = full2.values[mask].reshape(mask.sum(), 1)
full2_new = io.copy_xarray_template(full2)
full2_new.values[mask] = lm.predict(X)
full2_new.values[mask][full2_new.values[mask] < 0] = 0
outfile_prefix = ('%s%s2_band%i_corrected' %
(path2processed, site_id, band))
io.write_xarray_to_GeoTiff(full2_new, outfile_prefix)
# use gdal to merge and warp to extent
os.system(
"gdal_merge.py -a_nodata -9999 -ot float32 -o %s%s_b%s_temp.tif %s%s2_band%i_corrected.tif %s%s1_band%i.tif"
% (path2final, site_id, band, path2processed, site_id, band, path2full,
site_id, band))
os.system(
"gdalwarp -overwrite -te %f %f %f %f -tr %f %f -r near %s%s_b%i_temp.tif %s%s_b%i_10m.tif"
% (W, S, E, N, xres, yres, path2final, site_id, band, path2final,
site_id, band))
os.system("rm %s%s_b%i_temp.tif" % (path2final, site_id, band))
os.system("chmod +777 %s%s_b%i_10m.tif" % (path2final, site_id, band))
# calculate _ndvi# ndvi = (nir-red)/(nir+red)
red = xr.open_rasterio('%s%s_b3_10m.tif' % (path2final, site_id))[0]
agb_stack = np.zeros((N_iter,rows,cols))
#pca_predictors = pca.transform(predictors)
#predictors=None
for ii, agb_file in enumerate(agb_list):
print('Iteration %i of %i' % (ii+1,N_iter))
rf_dict = joblib.load('%s%s_%s_optimised_rfbc_sentinel_alos_lidar_%s.pkl' % (path2alg,site_id,version,str(ii+1).zfill(3)))
agb_mod = rff.rfbc_predict(rf_dict['rf1'],rf_dict['rf2'],predictors)
#let's copy to a new xarray for AGBpot
agb = io.copy_xarray_template(template)
agb.values[forest_mask] = agb_mod.copy()
agb.values[agb.values==-9999]=np.nan
agb.values[agb.values<0]=0
outfile_prefix = '%s%s_%s_rfbc_agb_upscaled_%s' % (path2output,site_id,version,str(ii+1).zfill(3))
io.write_xarray_to_GeoTiff(agb,outfile_prefix)
agb_stack[ii] = agb.values
# summary arrays
agb_med = io.copy_xarray_template(template)
agb_med.values = np.median(agb_stack,axis=0)
agb_med.values[agb_med.values==-9999]=np.nan
outfile_prefix = '%s%s_%s_rfbc_agb_upscaled_median' % (path2output,site_id,version)
io.write_xarray_to_GeoTiff(agb_med,outfile_prefix)
agb_upper = io.copy_xarray_template(template)
agb_upper.values = np.percentile(agb_stack,97.5,axis=0)
agb_upper.values[agb_upper.values==-9999]=np.nan
outfile_prefix = '%s%s_%s_rfbc_agb_upscaled_upper' % (path2output,site_id,version)
io.write_xarray_to_GeoTiff(agb_upper,outfile_prefix)
print('Generating reference maps from lidar data')
# load DSM and DTM
dtm = xr.open_rasterio(dtm_file).sel(band=1)
dsm = xr.open_rasterio(dsm_file).sel(band=1)
dtm.values[dtm.values == -9999.] = np.nan
dsm.values[dsm.values == -9999.] = np.nan
# TCH
tch = dsm - dtm
tch.values[tch.values < 0] = 0
# Cover at 2m height
cov_2m = tch.copy(deep=True)
cov_2m.values = (tch.values >= 2).astype('float')
cov_2m.values[np.isnan(tch.values)] = np.nan
io.write_xarray_to_GeoTiff(cov_2m, '%s_cover2m_1m' % site)
# - Cover at 0.1m height
cov_10cm = tch.copy(deep=True)
cov_10cm.values = (tch.values >= 0.1).astype('float')
cov_10cm.values[np.isnan(tch.values)] = np.nan
io.write_xarray_to_GeoTiff(cov_10cm, '%s_cover10cm_1m' % site)
# Identify "trees"
# - define as contiguous regions with canopy cover >2m comprising >8 pixels
# (~3m diameter)
# - use two-step procedure
# (i) fill "holes"
# (ii) remove objects based on connectivity direct connections in either
# row or column (ignoring diagonals)
trees = cov_2m.copy(deep=True)