def preprocess_dem(dem_path, raster_path, out_directory):
in_raster = gdal.Open(raster_path)
dem_name = p.basename(dem_path).partition('.')[0]
clipped_dem_path = p.join(out_directory, "{}_clipped.tif".format(dem_name))
reproj_dem_path = p.join(out_directory, "{}_reproj.tif".format(dem_name))
ras.reproject_image(dem_path,
reproj_dem_path,
in_raster.GetProjection(),
do_post_resample=False)
ras.resample_image_in_place(
reproj_dem_path,
in_raster.GetGeoTransform()[1]) # Assuming square pixels
ras.align_image_in_place(reproj_dem_path, raster_path)
ras.clip_raster_to_intersection(reproj_dem_path, raster_path,
clipped_dem_path)
return clipped_dem_path
def create_seg_tif(working_dir,
search_suffix='.tif',
export_pre_seg_tif=False,
export_brightness=True):
os.chdir(working_dir)
with TemporaryDirectory() as td:
for image in os.listdir("composites/10m"):
if image.endswith(search_suffix):
image_path_10m = os.path.join("composites/10m", image)
image_path_20m = os.path.join("composites/20m", image)
resample_path_20m = os.path.join(
td, image) # You'll probably regret this later, roberts.
shutil.copy(image_path_20m, resample_path_20m)
ras.resample_image_in_place(resample_path_20m, 10)
# Now, we do Highly Experimental Image Segmentation. Please put on your goggles.
# SAGA, please.
# Meatball science time
vis_10m = gdal.Open(image_path_10m)
vis_20m_resampled = gdal.Open(resample_path_20m)
vis_10m_array = vis_10m.GetVirtualMemArray()
vis_20m_array = vis_20m_resampled.GetVirtualMemArray()
# NIR, SWIR, red
array_to_classify = np.stack([
vis_10m_array[3, ...], vis_20m_array[7, ...],
vis_10m_array[2, ...]
])
temp_pre_seg_path = os.path.join(td, "pre_seg.tif")
#temp_seg_path = os.path.join("seg.tif")
# shape_projection = osr.SpatialReference()
# shape_projection.ImportFromWkt(vis_10m.GetProjection())
# image_gt = vis_10m.GetGeoTransform()
# ras.save_array_as_image(array_to_classify, temp_pre_seg_path, image_gt, shape_projection)
g, arr = read_tif(intif=image_path_10m)
create_tif(filename=temp_pre_seg_path,
g=g,
Nx=arr.shape[1],
Ny=arr.shape[2],
new_array=array_to_classify,
noData=0,
data_type=gdal.GDT_UInt32)
out_seg_tif = os.path.join("segmentation", image)
#segment_image(temp_pre_seg_path, out_seg_tif)
if export_pre_seg_tif == True:
shutil.copy(temp_pre_seg_path,
os.path.join("pre_segmentation", image))
if export_brightness == True:
in_value_ras = temp_pre_seg_path
seg_ras = out_seg_tif
try:
os.mkdir("segmentation/brightness")
except FileExistsError:
pass
out_value_ras = os.path.join("segmentation/brightness",
image)
output_filtered_value_ras = False
temp_reshape_for_brightness = os.path.join(
td, "pre_brightness.tif")
clip_rst_by_rst(in_tif=seg_ras,
ref_tif=in_value_ras,
out_tif=temp_reshape_for_brightness)
cal_seg_mean(
in_value_ras,
temp_reshape_for_brightness,
out_value_ras,
output_filtered_value_ras=output_filtered_value_ras)
def build_cocoa_map(working_dir,
path_to_aoi,
start_date,
end_date,
path_to_s1_image,
path_to_config,
epsg_for_map,
path_to_model,
cloud_cover=20,
log_path="build_cocoa_map.log",
use_sen2cor=False,
sen2cor_path=None,
skip_download_and_preprocess=False,
skip_composite=False):
# Step 0: Get things ready. Folder structure for downloads, load credentials from config.
fu.init_log(log_path)
os.chdir(working_dir)
# fu.create_file_structure(os.getcwd())
#
# make_directory("images/merged/10m")
# make_directory("images/merged/20m")
#
# make_directory("images/stacked/with_indices")
# make_directory("images/stacked/with_s1_seg")
# make_directory("images/stacked/all_19bands")
#
# make_directory("segmentation")
# make_directory("composites")
# make_directory("composites/10m")
# make_directory("composites/20m")
#
# make_directory("composites/10m_full")
# make_directory("composites/20m_full")
general_functions.make_all_dirs(working_dir)
# Step 1: Download S2 3imagery for the timescale
if not skip_download_and_preprocess:
config = configparser.ConfigParser()
config.read(path_to_config)
images_to_download = query.check_for_s2_data_by_date(
path_to_aoi, start_date, end_date, config, cloud_cover)
if not use_sen2cor:
images_to_download = query.filter_non_matching_s2_data(
images_to_download)
else:
images_to_download = query.filter_to_l1_data(images_to_download)
query.download_s2_data(images_to_download, "images/L1", "images/L2")
# Step 2: Preprocess S2 imagery. Perform atmospheric correction if needed, stack and mask 10 and 20m bands.
if use_sen2cor:
ras.atmospheric_correction("images/L1"
"images/L2",
sen2cor_path=sen2cor_path)
ras.preprocess_sen2_images("images/L2",
"images/merged/10m",
"images/L1",
cloud_threshold=0,
epsg=epsg_for_map,
bands=("B02", "B03", "B04", "B08"),
out_resolution=10)
ras.preprocess_sen2_images("images/L2",
"images/merged/20m",
"images/L1",
cloud_threshold=0,
epsg=epsg_for_map,
bands=("B02", "B03", "B04", "B05", "B06",
"B07", "B8A", "B11", "B12"),
out_resolution=20)
if not skip_composite:
# Step 2.5: Build a pair of cloud-free composites
sort_into_tile("images/merged/10m")
sort_into_tile("images/merged/20m")
for tile in os.listdir("images/merged/10m"):
tile_path = os.path.join("images/merged/10m", tile)
this_composite_path = ras.composite_directory(
tile_path, "composites/10m")
new_composite_path = "{}_{}.tif".format(
this_composite_path.rsplit('.')[0], tile)
os.rename(this_composite_path, new_composite_path)
for tile in os.listdir("images/merged/20m"):
tile_path = os.path.join("images/merged/20m", tile)
this_composite_path = ras.composite_directory(
tile_path, "composites/20m")
new_composite_path = "{}_{}.tif".format(
this_composite_path.rsplit('.')[0], tile)
os.rename(this_composite_path, new_composite_path)
# Step 3: Generate the bands. Time for the New Bit.
clip_to_aoi = False
if clip_to_aoi:
for image in os.listdir("composites/10m_full"):
if image.endswith(".tif"):
image_path_10m_full = os.path.join("composites/10m_full",
image)
image_path_20m_full = os.path.join("composites/20m_full",
image)
image_path_10m_clipped = os.path.join("composites/10m", image)
image_path_20m_clipped = os.path.join("composites/20m", image)
# config = configparser.ConfigParser()
# conf = config.read(path_to_config)
# print(conf)
# aoi = config['cocoa_mapping']['path_to_aoi']
aoi = "/media/ubuntu/Data/Ghana/cocoa_big/shp/cocoa_big.shp"
ras.clip_raster(raster_path=image_path_10m_full,
aoi_path=aoi,
out_path=image_path_10m_clipped,
srs_id=32630)
ras.clip_raster(raster_path=image_path_20m_full,
aoi_path=aoi,
out_path=image_path_20m_clipped,
srs_id=32630)
do_segmentation = True
if do_segmentation == True:
for image in os.listdir("composites/10m"):
if image.endswith(".tif"):
with TemporaryDirectory() as td:
image_path_10m = os.path.join("composites/10m", image)
image_path_20m = os.path.join("composites/20m", image)
resample_path_20m_v1 = os.path.join(
td,
image) # You'll probably regret this later, roberts.
shutil.copy(image_path_20m, resample_path_20m_v1)
ras.resample_image_in_place(resample_path_20m_v1, 10)
index_image_path = os.path.join(td, "index_image.tif")
temp_pre_seg_path = os.path.join(td, "pre_seg.tif")
temp_seg_path = os.path.join(td, "seg.tif")
temp_shp_path = os.path.join(td, "outline.shp")
temp_clipped_seg_path = os.path.join(td, "seg_clip.tif")
# This bit's your show, Qing
temp_s1_outline_path = os.path.join(td, "s1_outline.shp")
ras.get_extent_as_shp(in_ras_path=image_path_10m,
out_shp_path=temp_s1_outline_path)
resample_path_20m = os.path.join(
td, image[:-4] + '_to_10moutline.tif')
general_functions.clip_rst(
in_tif=resample_path_20m_v1,
outline_shp=temp_s1_outline_path,
out_tif=resample_path_20m,
keep_rst_extent=False)
generate_veg_index_tif(image_path_10m, resample_path_20m,
index_image_path)
ras.stack_images([index_image_path, image_path_10m],
"images/stacked/with_indices/" + image)
# Now, we do Highly Experimental Image Segmentation. Please put on your goggles.
# SAGA, please.
# Meatball science time
vis_10m = gdal.Open(image_path_10m)
vis_20m_resampled = gdal.Open(resample_path_20m)
vis_10m_array = vis_10m.GetVirtualMemArray()
vis_20m_array = vis_20m_resampled.GetVirtualMemArray()
# NIR, SWIR, red
array_to_classify = np.stack([
vis_10m_array[3, ...], vis_20m_array[7, ...],
vis_10m_array[2, ...]
])
g, arr = general_functions.read_tif(intif=image_path_10m)
general_functions.create_tif(filename=temp_pre_seg_path,
g=g,
Nx=arr.shape[1],
Ny=arr.shape[2],
new_array=array_to_classify,
noData=0,
data_type=gdal.GDT_UInt32)
out_segment_tif = os.path.join("segmentation", image)
segment_image(temp_pre_seg_path, out_segment_tif)
print('Generate brighness raster from the segments')
make_directory("segmentation/brightness")
out_brightness_value_ras = os.path.join(
"segmentation/brightness", image)
output_filtered_value_ras = False
ras.get_extent_as_shp(in_ras_path=temp_pre_seg_path,
out_shp_path=temp_shp_path)
general_functions.clip_rst(in_tif=out_segment_tif,
outline_shp=temp_shp_path,
out_tif=temp_clipped_seg_path,
keep_rst_extent=False)
cal_seg_mean(
temp_pre_seg_path,
temp_clipped_seg_path,
out_brightness_value_ras,
output_filtered_value_ras=output_filtered_value_ras)
# image_20m_6bands_array = vis_20m_array[3:,:,:]
# try:
# os.mkdir("composites/20m/20m_6bands")
# except FileExistsError:
# pass
#
# out_20m_tif_for_stack = os.path.join("composites/20m/20m_6bands", image)
# general_functions.create_tif(filename=out_20m_tif_for_stack,g=g,Nx=arr.shape[1],Ny=arr.shape[2],
# new_array=image_20m_6bands_array,data_type=gdal.GDT_UInt16,noData=0)
do_stack = True
if do_stack == True:
# Step 4: Stack the new bands with the S1, seg, and 6 band 20m rasters
for image in os.listdir("images/stacked/with_indices"):
if image.endswith(".tif"):
path_to_image = os.path.join("images/stacked/with_indices",
image)
path_to_brightness_image = os.path.join(
"segmentation/brightness", image)
# path_to_20m_image = os.path.join("composites/20m/20m_6bands", image)
# ras.stack_images([path_to_image, path_to_s1_image,path_to_20m_image,out_brightness_value_ras], os.path.join("images/stacked/all_19bands", image))
ras.stack_images([
path_to_image, path_to_s1_image, path_to_brightness_image
], os.path.join("images/stacked/with_s1_seg", image))
#sys.exit()
#
# Step 5: Classify with trained model
for image in os.listdir("images/stacked/with_s1_seg"):
if image.endswith(".tif"):
path_to_image = os.path.join("images/stacked/with_s1_seg", image)
path_to_out = os.path.join("output", image)
PYEO_model.classify_image(path_to_image, path_to_model,
path_to_out)
def calculate_reflectance(raster_path, dem_path, out_raster_path, raster_datetime, is_landsat = False):
"""
Corrects for shadow effects due to terrain features.
Algorithm:
* Generate slope and aspect from DEM using gdaldem
* Calculate solar position from datatake sensing start and location of image
* Calculate the correction factor for that image from the sun zenith angle, azimuth angle, DEM aspect and DEM slope
* Build a mask of green areas using NDVI
* Perform a linear regression based on that IC calculation and the contents of the L2 image to get ground slope(?)
* Correct pixel p in original image with following: p_out = p_in - (ground_slope*(IC-cos(sun_zenith)))
* Write to output
Parameters
----------
raster_path
A path to the raster to correct.
dem_path
The path to the DEM
out_raster_path
The path to the output.
raster_datetime
A datetime.DateTime object **with timezone set**
"""
with TemporaryDirectory() as td:
in_raster = gdal.Open(raster_path)
in_array = in_raster.GetVirtualMemArray()
out_raster = ras.create_matching_dataset(in_raster, out_raster_path, bands=in_raster.RasterCount)
out_array = out_raster.GetVirtualMemArray(eAccess=gdal.GA_Update)
print("Preprocessing DEM")
clipped_dem_path = p.join(td, "clipped_dem.tif")
reproj_dem_path = p.join(td, "reproj_dem.tif")
ras.reproject_image(dem_path, reproj_dem_path, in_raster.GetProjection(), do_post_resample=False)
ras.resample_image_in_place(reproj_dem_path, in_raster.GetGeoTransform()[1]) # Assuming square pixels
ras.clip_raster_to_intersection(reproj_dem_path, raster_path, clipped_dem_path, is_landsat)
ic_array, zenith_array, slope_array = calculate_illumination_condition_array(clipped_dem_path, raster_datetime)
if is_landsat:
in_array = in_array.T
if len(in_array.shape) == 2:
in_array = np.expand_dims(in_array, 0)
print("Calculating reflectance array")
# Oh no, magic numbers. I think these were from the original paper? Are they for Landsat?
ref_multi_this_band = 2.0e-5
ref_add_this_band = -0.1
ref_array = (ref_multi_this_band * in_array + ref_add_this_band) / _deg_cos(zenith_array.T)
print("Calculating sample array")
sample_array = build_sample_array(ref_array, slope_array, 2, 3) # THIS IS LANDAST ONLY RIGHT NOW, CHANGE LATER!
band_indicies = sample_array[0, ...].nonzero()
print("Beginning linear regression")
for i, band in enumerate(sample_array[:, ...]):
print("Processing band {} of {}".format(i+1, ref_array.shape[0]))
ic_for_linregress = ic_array.T[band_indicies[0], band_indicies[1]].ravel()
band_for_linregress = band[band_indicies[0], band_indicies[1]].ravel()
slope, _, _, _, _ = stats.linregress(ic_for_linregress, band_for_linregress)
corrected_band = (band - (slope*(ic_array.T - _deg_cos(zenith_array.T))))
out_array[i, ...] = np.where(band > 0, corrected_band, ref_array[i, ...])
out_array = None
out_raster = None
ref_array = None
in_raster = None