def generate_veg_index_tif(tif_10m, tif_20m, out_tif):
g_10m, arr_10m = general_functions.read_tif(intif=tif_10m, type=np.int32)
g_20m, arr_20m = general_functions.read_tif(intif=tif_20m, type=np.int32)
veg_arr = cal_vegIndex(arr_10m, arr_20m)
veg_array_trans = np.transpose(veg_arr, (2, 0, 1))
general_functions.create_tif(filename=out_tif,
g=g_10m,
Nx=arr_20m.shape[1],
Ny=arr_20m.shape[2],
new_array=veg_array_trans,
noData=0,
data_type=gdal.GDT_Int32)
def generate_seg_rst():
'''
:param s0:array read from s2 20m 9 bands data .tif or 10m 4 bands data.tif
for s2 20m, the band sequence are: band 2, 3, 4, 5, 6, 7, 8a, 11, 12
for s2 10m, the band sequence are: band 2, 3, 4, 8
:return:
'''
merge_10m_dir = "/media/ubuntu/Data/Ghana/north_region/s2/composites/10m/"
merge_20m_dir = "/media/ubuntu/Data/Ghana/north_region/s2/composites/20m/"
tif_10m_list = s2_functions.search_files_fulldir(input_path=merge_10m_dir,
search_type='end',
search_key='NWM.tif')
tif_20m_list = s2_functions.search_files_fulldir(input_path=merge_20m_dir,
search_type='end',
search_key='NWM.tif')
for n in range(len(tif_10m_list)):
tif_10m = tif_10m_list[n]
tif_20m = tif_20m_list[n]
print(tif_10m)
print(tif_20m)
out_tif = tif_10m[:-4] + "_NIR_SWIR_red.tif"
g_10m, a_10m = general_functions.read_tif(tif_10m)
a_10m_trans = np.transpose(a_10m, (1, 2, 0))
g_20m, a_20m = general_functions.read_tif(tif_20m)
a_20m_trans = np.transpose(a_20m, (1, 2, 0))
a_3band = np.zeros((a_10m_trans.shape[0], a_10m_trans.shape[1], 3),
dtype=int)
# # # version 3: NIR, SWIR, and red
a_3band[:, :, 0] = a_10m_trans[:, :, 3]
a_3band[:, :, 1] = a_20m_trans[:, :, 7]
a_3band[:, :, 2] = a_10m_trans[:, :, 2]
a_3band_trans = np.transpose(a_3band, (2, 0, 1))
a_3band_out = tif_10m[:-4] + '_NIR_SWIR_red.tif'
general_functions.create_tif(filename=a_3band_out,
g=g_20m,
Nx=a_20m.shape[1],
Ny=a_20m.shape[2],
new_array=a_3band_trans,
noData=0,
data_type=gdal.GDT_UInt16)
def classify_image(in_image_path,
model_path,
out_image_path,
num_chunks=10,
rescale_predict_image=None,
ref_img_for_linear_shift=None,
generate_mask=True):
print("Using model: " + model_path)
model = joblib.load(model_path)
if generate_mask:
image, image_array = general_functions.read_tif(in_image_path)
print("Generating extent mask for mosaicing: ")
mask_array = image_array[6, :, :]
mask_array[mask_array != 0] = 1
general_functions.create_tif(filename=in_image_path[:-4] + '_mask.msk',
g=image,
Nx=mask_array.shape[0],
Ny=mask_array.shape[1],
new_array=mask_array,
data_type=gdal.GDT_UInt16,
noData=0)
else:
image = gdal.Open(in_image_path)
image_array = image.GetVirtualMemArray()
features_to_classify = reshape_raster_for_ml(image_array)
width = image.RasterXSize
height = image.RasterYSize
if ref_img_for_linear_shift is not None:
diff_array = cal_linear_shift_array(ref_tif=ref_img_for_linear_shift,
tobe_shift_tif=in_image_path)
features_to_classify_filtered = features_to_classify + diff_array
if rescale_predict_image is not None:
features_to_classify_norm = rescale_predict_image.transform(
features_to_classify_filtered)
else:
features_to_classify_norm = features_to_classify_filtered
print("Classifying image")
out_chunks = []
for i, chunk in enumerate(
np.array_split(features_to_classify_norm, num_chunks)):
print("Classifying {0}".format(i))
chunk_copy = np.copy(chunk)
chunk_copy = np.where(np.isfinite(chunk_copy), chunk_copy,
0) # this is a slower line
out_chunks.append(model.predict(chunk_copy))
out_classes = np.concatenate(out_chunks)
image = gdal.Open(in_image_path)
out_image = create_matching_dataset(image, out_image_path)
image_array = None
image = None
out_image_array = out_image.GetVirtualMemArray(eAccess=gdal.GA_Update)
out_image_array[...] = reshape_ml_out_to_raster(out_classes, width, height)
out_image_array = None
out_image = None
def cal_seg_mean(in_value_ras,
seg_ras,
out_value_ras,
output_filtered_value_ras=True):
g_seg, seg_arr = general_functions.read_tif(seg_ras, type=np.int64)
g_value, value_arr = general_functions.read_tif(in_value_ras,
type=np.int64)
out_array = np.zeros(value_arr.shape)
unique_segvals = np.unique(seg_arr)
band_number = value_arr.shape[0]
for n in range(band_number):
print('working on band: ' + str(n + 1))
band_val = value_arr[n, :, :]
mean_arr = scipy.ndimage.measurements.mean(band_val,
labels=seg_arr,
index=unique_segvals)
# new_arr = np.zeros(band_val.shape)
lookup_array = np.zeros(unique_segvals.max() + 1)
lookup_array[unique_segvals] = mean_arr
new_arr = lookup_array[seg_arr]
out_array[n, :, :] = new_arr
if output_filtered_value_ras:
general_functions.create_tif(filename=out_value_ras,
g=g_seg,
Nx=seg_arr.shape[0],
Ny=seg_arr.shape[1],
new_array=out_array,
data_type=gdal.GDT_UInt32,
noData=0)
else:
print('Caculating brightness layer...')
brightness = np.mean(out_array, axis=0)
print(brightness.shape)
general_functions.create_tif(filename=out_value_ras,
g=g_seg,
Nx=seg_arr.shape[0],
Ny=seg_arr.shape[1],
new_array=brightness,
data_type=gdal.GDT_UInt32,
noData=0)
def generate_20m_6bands(in_20m_tif):
g, arr = general_functions.read_tif(in_20m_tif)
out_arr = arr[3:, :, :]
filename = in_20m_tif[:-4] + '_6bands.tif'
general_functions.create_tif(filename=filename,
g=g,
Nx=arr.shape[1],
Ny=arr.shape[2],
new_array=out_arr,
data_type=gdal.GDT_Int16,
noData=0)
def cal_linear_shift_array(ref_tif, tobe_shift_tif):
# training_tif = "/media/ubuntu/Data/Ghana/cocoa_upscale_test/all_13bands_stack_v2.tif"
# tobe_shift_tif = "/media/ubuntu/Data/Ghana/north_region/s2_NWN/images/stacked/with_s1_seg/composite_20180122T102321_T30NWN.tif"
g_ref, a_ref = general_functions.read_tif(ref_tif, type=np.float)
a_ref_ml = reshape_raster_for_ml(a_ref)
a_ref = None
median_a_ref = hist_matching_plotting.cal_non_zero_median(a_ref_ml)
a_ref_ml = None
g_tobe_shift, a_tobe_shift = general_functions.read_tif(tobe_shift_tif,
type=np.float)
a_tobe_shift_ml = reshape_raster_for_ml(a_tobe_shift)
a_tobe_shift = None
median_tobe_shift = hist_matching_plotting.cal_non_zero_median(
a_tobe_shift_ml)
diff = np.array(median_a_ref - median_tobe_shift)
return diff
a_tobe_shift_ml = None
def build_cocoa_map(working_dir,
path_to_aoi,
start_date,
end_date,
path_to_s1_image,
path_to_config,
epsg_for_map,
path_to_model,
cloud_cover=20,
log_path="build_cocoa_map.log",
use_sen2cor=False,
sen2cor_path=None,
skip_download_and_preprocess=False,
skip_composite=False):
# Step 0: Get things ready. Folder structure for downloads, load credentials from config.
fu.init_log(log_path)
os.chdir(working_dir)
# fu.create_file_structure(os.getcwd())
#
# make_directory("images/merged/10m")
# make_directory("images/merged/20m")
#
# make_directory("images/stacked/with_indices")
# make_directory("images/stacked/with_s1_seg")
# make_directory("images/stacked/all_19bands")
#
# make_directory("segmentation")
# make_directory("composites")
# make_directory("composites/10m")
# make_directory("composites/20m")
#
# make_directory("composites/10m_full")
# make_directory("composites/20m_full")
general_functions.make_all_dirs(working_dir)
# Step 1: Download S2 3imagery for the timescale
if not skip_download_and_preprocess:
config = configparser.ConfigParser()
config.read(path_to_config)
images_to_download = query.check_for_s2_data_by_date(
path_to_aoi, start_date, end_date, config, cloud_cover)
if not use_sen2cor:
images_to_download = query.filter_non_matching_s2_data(
images_to_download)
else:
images_to_download = query.filter_to_l1_data(images_to_download)
query.download_s2_data(images_to_download, "images/L1", "images/L2")
# Step 2: Preprocess S2 imagery. Perform atmospheric correction if needed, stack and mask 10 and 20m bands.
if use_sen2cor:
ras.atmospheric_correction("images/L1"
"images/L2",
sen2cor_path=sen2cor_path)
ras.preprocess_sen2_images("images/L2",
"images/merged/10m",
"images/L1",
cloud_threshold=0,
epsg=epsg_for_map,
bands=("B02", "B03", "B04", "B08"),
out_resolution=10)
ras.preprocess_sen2_images("images/L2",
"images/merged/20m",
"images/L1",
cloud_threshold=0,
epsg=epsg_for_map,
bands=("B02", "B03", "B04", "B05", "B06",
"B07", "B8A", "B11", "B12"),
out_resolution=20)
if not skip_composite:
# Step 2.5: Build a pair of cloud-free composites
sort_into_tile("images/merged/10m")
sort_into_tile("images/merged/20m")
for tile in os.listdir("images/merged/10m"):
tile_path = os.path.join("images/merged/10m", tile)
this_composite_path = ras.composite_directory(
tile_path, "composites/10m")
new_composite_path = "{}_{}.tif".format(
this_composite_path.rsplit('.')[0], tile)
os.rename(this_composite_path, new_composite_path)
for tile in os.listdir("images/merged/20m"):
tile_path = os.path.join("images/merged/20m", tile)
this_composite_path = ras.composite_directory(
tile_path, "composites/20m")
new_composite_path = "{}_{}.tif".format(
this_composite_path.rsplit('.')[0], tile)
os.rename(this_composite_path, new_composite_path)
# Step 3: Generate the bands. Time for the New Bit.
clip_to_aoi = False
if clip_to_aoi:
for image in os.listdir("composites/10m_full"):
if image.endswith(".tif"):
image_path_10m_full = os.path.join("composites/10m_full",
image)
image_path_20m_full = os.path.join("composites/20m_full",
image)
image_path_10m_clipped = os.path.join("composites/10m", image)
image_path_20m_clipped = os.path.join("composites/20m", image)
# config = configparser.ConfigParser()
# conf = config.read(path_to_config)
# print(conf)
# aoi = config['cocoa_mapping']['path_to_aoi']
aoi = "/media/ubuntu/Data/Ghana/cocoa_big/shp/cocoa_big.shp"
ras.clip_raster(raster_path=image_path_10m_full,
aoi_path=aoi,
out_path=image_path_10m_clipped,
srs_id=32630)
ras.clip_raster(raster_path=image_path_20m_full,
aoi_path=aoi,
out_path=image_path_20m_clipped,
srs_id=32630)
do_segmentation = True
if do_segmentation == True:
for image in os.listdir("composites/10m"):
if image.endswith(".tif"):
with TemporaryDirectory() as td:
image_path_10m = os.path.join("composites/10m", image)
image_path_20m = os.path.join("composites/20m", image)
resample_path_20m_v1 = os.path.join(
td,
image) # You'll probably regret this later, roberts.
shutil.copy(image_path_20m, resample_path_20m_v1)
ras.resample_image_in_place(resample_path_20m_v1, 10)
index_image_path = os.path.join(td, "index_image.tif")
temp_pre_seg_path = os.path.join(td, "pre_seg.tif")
temp_seg_path = os.path.join(td, "seg.tif")
temp_shp_path = os.path.join(td, "outline.shp")
temp_clipped_seg_path = os.path.join(td, "seg_clip.tif")
# This bit's your show, Qing
temp_s1_outline_path = os.path.join(td, "s1_outline.shp")
ras.get_extent_as_shp(in_ras_path=image_path_10m,
out_shp_path=temp_s1_outline_path)
resample_path_20m = os.path.join(
td, image[:-4] + '_to_10moutline.tif')
general_functions.clip_rst(
in_tif=resample_path_20m_v1,
outline_shp=temp_s1_outline_path,
out_tif=resample_path_20m,
keep_rst_extent=False)
generate_veg_index_tif(image_path_10m, resample_path_20m,
index_image_path)
ras.stack_images([index_image_path, image_path_10m],
"images/stacked/with_indices/" + image)
# Now, we do Highly Experimental Image Segmentation. Please put on your goggles.
# SAGA, please.
# Meatball science time
vis_10m = gdal.Open(image_path_10m)
vis_20m_resampled = gdal.Open(resample_path_20m)
vis_10m_array = vis_10m.GetVirtualMemArray()
vis_20m_array = vis_20m_resampled.GetVirtualMemArray()
# NIR, SWIR, red
array_to_classify = np.stack([
vis_10m_array[3, ...], vis_20m_array[7, ...],
vis_10m_array[2, ...]
])
g, arr = general_functions.read_tif(intif=image_path_10m)
general_functions.create_tif(filename=temp_pre_seg_path,
g=g,
Nx=arr.shape[1],
Ny=arr.shape[2],
new_array=array_to_classify,
noData=0,
data_type=gdal.GDT_UInt32)
out_segment_tif = os.path.join("segmentation", image)
segment_image(temp_pre_seg_path, out_segment_tif)
print('Generate brighness raster from the segments')
make_directory("segmentation/brightness")
out_brightness_value_ras = os.path.join(
"segmentation/brightness", image)
output_filtered_value_ras = False
ras.get_extent_as_shp(in_ras_path=temp_pre_seg_path,
out_shp_path=temp_shp_path)
general_functions.clip_rst(in_tif=out_segment_tif,
outline_shp=temp_shp_path,
out_tif=temp_clipped_seg_path,
keep_rst_extent=False)
cal_seg_mean(
temp_pre_seg_path,
temp_clipped_seg_path,
out_brightness_value_ras,
output_filtered_value_ras=output_filtered_value_ras)
# image_20m_6bands_array = vis_20m_array[3:,:,:]
# try:
# os.mkdir("composites/20m/20m_6bands")
# except FileExistsError:
# pass
#
# out_20m_tif_for_stack = os.path.join("composites/20m/20m_6bands", image)
# general_functions.create_tif(filename=out_20m_tif_for_stack,g=g,Nx=arr.shape[1],Ny=arr.shape[2],
# new_array=image_20m_6bands_array,data_type=gdal.GDT_UInt16,noData=0)
do_stack = True
if do_stack == True:
# Step 4: Stack the new bands with the S1, seg, and 6 band 20m rasters
for image in os.listdir("images/stacked/with_indices"):
if image.endswith(".tif"):
path_to_image = os.path.join("images/stacked/with_indices",
image)
path_to_brightness_image = os.path.join(
"segmentation/brightness", image)
# path_to_20m_image = os.path.join("composites/20m/20m_6bands", image)
# ras.stack_images([path_to_image, path_to_s1_image,path_to_20m_image,out_brightness_value_ras], os.path.join("images/stacked/all_19bands", image))
ras.stack_images([
path_to_image, path_to_s1_image, path_to_brightness_image
], os.path.join("images/stacked/with_s1_seg", image))
#sys.exit()
#
# Step 5: Classify with trained model
for image in os.listdir("images/stacked/with_s1_seg"):
if image.endswith(".tif"):
path_to_image = os.path.join("images/stacked/with_s1_seg", image)
path_to_out = os.path.join("output", image)
PYEO_model.classify_image(path_to_image, path_to_model,
path_to_out)