def do_gridSearch(training_shp, training_image, model_out_path, bands):
PYEO_model.train_cairan_model(image_path=training_image,
shp_path=training_shp,
outModel_path=model_out_path,
bands=bands,
attribute='Id',
shape_projection_id=32630)
def test_do_classify_image():
image_to_classify = "/media/ubuntu/Data/Ghana/cocoa_upscale_test/all_19bands_stack.tif"
out_image_path = image_to_classify[:-4] + '_classified_v2.tif'
model_path = image_to_classify[:-4] + '.pkl'
PYEO_model.classify_image(model_path=model_path,
in_image_path=image_to_classify,
out_image_path=out_image_path)
def test_histmatching_classify():
training_tif = "/media/ubuntu/Data/Ghana/cocoa_upscale_test/all_13bands_stack_v2.tif"
george_points_tif = "/media/ubuntu/Data/Ghana/cocoa_upscale_test/shp/Training_points_clip_dropTransition_add.tif"
input_model = "/media/ubuntu/Data/Ghana/cocoa_upscale_test/george_data_13bands_add_255_eRF_v2.pkl"
image_tobe_classified = "/media/ubuntu/Data/Ghana/north_region/s2/images/stacked/with_s1_seg/composite_20180122T102321_T30NWN.tif"
features, labels = PYEO_model.get_training_data_tif(
image_path=training_tif, training_tif_path=george_points_tif)
scale = preprocessing.MinMaxScaler(feature_range=(0, 255)).fit(features)
PYEO_model.classify_image(model_path=input_model,
in_image_path=image_tobe_classified,
out_image_path=image_tobe_classified[:-4] +
"_classified_13bands_255_eRF2_v4.tif",
rescale_predict_image=scale,
ref_img_for_linear_shift=training_tif,
generate_mask=False)
def train_RS():
george_points_tif = "/media/ubuntu/Data/Ghana/cocoa_upscale_test/shp/Training_points_clip_dropTransition_add.tif"
# training_tif = "/media/ubuntu/Data/Ghana/cocoa_upscale_test/all_19bands_stack.tif"
# outmodel = "/media/ubuntu/Data/Ghana/cocoa_upscale_test/george_data_19bands_add.pkl"
# training_summary = "/media/ubuntu/Data/Ghana/cocoa_upscale_test/george_data_19bands_add.csv"
# band_labels = ["ndvi", "ci", "psri", "gndvi", "s2_rep", "ireci", "s2_b", "s2_g",
# "s2_r", "s2_nir", "hv", "vv","s2_20m_1","s2_20m_2","s2_20m_3","s2_20m_4","s2_20m_5","s2_20m_6","seg"]
#bands = 19
training_tif = "/media/ubuntu/Data/Ghana/cocoa_upscale_test/all_13bands_stack_v2.tif"
outmodel = "/media/ubuntu/Data/Ghana/cocoa_upscale_test/george_data_13bands_add_255_eRF_v2.pkl"
training_summary = "/media/ubuntu/Data/Ghana/cocoa_upscale_test/george_data_13bands_add_255_v2.csv"
band_labels = [
"ndvi", "ci", "psri", "gndvi", "s2_rep", "ireci", "s2_b", "s2_g",
"s2_r", "s2_nir", "hv", "vv", "seg"
]
bands = 13
features, labels = PYEO_model.get_training_data_tif(
image_path=training_tif, training_tif_path=george_points_tif)
features_df = pd.DataFrame(data=features,
columns=band_labels).astype(np.int32)
scale = preprocessing.MinMaxScaler(feature_range=(0, 255)).fit(features)
features_trans = scale.transform(features)
features_trans_df = pd.DataFrame(features_trans,
index=features_df.index,
columns=features_df.columns)
features_df.describe()
features_trans_df.describe()
labels_df = pd.DataFrame(data=labels,
columns=['class_name']).astype(np.int)
features_df["class_name"] = labels_df["class_name"]
features_df.describe().to_csv(training_summary)
#PYEO_model.train_rs_simple(X_train_input=features_trans,y_train_input=labels,outModel_path=outmodel, bands = bands)
PYEO_model.train_eRF_gs_simple(X_train_input=features_trans,
y_train_input=labels,
outModel_path=outmodel,
bands=bands)
image_tobe_classified = training_tif
PYEO_model.classify_image(model_path=outmodel,
in_image_path=image_tobe_classified,
out_image_path=image_tobe_classified[:-4] +
"_classified_13bands_255_eRF2_v2.tif",
rescale_predict_image=scale)
image_tobe_classified = "/media/ubuntu/Data/Ghana/north_region/s2_NWN/images/stacked/with_s1_seg/composite_20180122T102321_T30NWN.tif"
PYEO_model.classify_image(model_path=outmodel,
in_image_path=image_tobe_classified,
out_image_path=image_tobe_classified[:-4] +
"_classified_13bands_255_eRF2_v2.tif",
rescale_predict_image=scale)
def do_grid_search(
training_shape,
training_image,
image_out_path,
model_out_path,
):
PYEO_model.train_cairan_model(image_dir=s0_dir,
outModel_path=outModel,
bands=8,
attribute='Code')
features, classes = get_training_data(training_image,
training_shape,
attribute="Id")
print(features.shape)
print(classes.shape)
X_train, X_test, y_train, y_test = train_test_split(
features.astype(np.uint8),
classes.astype(np.uint8),
train_size=0.7,
test_size=0.3)
# grid search for the best parameters for SVM
# https://scikit-learn.org/stable/auto_examples/svm/plot_rbf_parameters.html
# C_range = np.logspace(0, 1, 2, base=10) # base = 2 for a fine tuning
# gamma_range = np.logspace(0, 128, 2, base=10)
# param_grid = dict(gamma=gamma_range, C=C_range)
# cv = StratifiedShuffleSplit(n_splits=5, test_size=0.2, random_state=42)
# grid = GridSearchCV(SVC(), param_grid=param_grid, cv=cv, n_jobs=6)
# grid.fit(X_train, y_train)
#
# print("The best parameters are %s with a score of %0.2f"
# % (grid.best_params_, grid.best_score_))
# c, gamma = grid.best_params_
# c=
# gamma =
model = svm.SVC(kernel='rbf')
# model = TPOTClassifier(generations=10, population_size=20, verbosity=2,n_jobs=-1)
model.fit(features, classes)
print(model.score(X_test, y_test))
model.export(model_out_path)
def do_classify(working_dir):
training_tif = "/media/ubuntu/Data/Ghana/cocoa_upscale_test/all_13bands_stack_v2.tif"
george_points_tif = "/media/ubuntu/Data/Ghana/cocoa_upscale_test/shp/Training_points_clip_dropTransition_add.tif"
input_model = "/media/ubuntu/Data/Ghana/cocoa_upscale_test/george_data_13bands_add_255_eRF_v2.pkl"
features, labels = PYEO_model.get_training_data_tif(
image_path=training_tif, training_tif_path=george_points_tif)
scale = preprocessing.MinMaxScaler(feature_range=(0, 255)).fit(features)
os.chdir(working_dir)
for image in os.listdir("images/stacked/with_s1_seg"):
if image.endswith(".tif"):
image_tobe_classified = os.path.join("images/stacked/with_s1_seg",
image)
out_image_path = os.path.join("output/",
image[:-4] + "_13bands_255_eRF.tif")
PYEO_model.classify_image(model_path=input_model,
in_image_path=image_tobe_classified,
out_image_path=out_image_path,
rescale_predict_image=scale,
ref_img_for_linear_shift=training_tif,
generate_mask=False)
def build_cocoa_map(working_dir,
path_to_aoi,
start_date,
end_date,
path_to_s1_image,
path_to_config,
epsg_for_map,
path_to_model,
cloud_cover=20,
log_path="build_cocoa_map.log",
use_sen2cor=False,
sen2cor_path=None,
skip_download_and_preprocess=False,
skip_composite=False):
# Step 0: Get things ready. Folder structure for downloads, load credentials from config.
fu.init_log(log_path)
os.chdir(working_dir)
# fu.create_file_structure(os.getcwd())
#
# make_directory("images/merged/10m")
# make_directory("images/merged/20m")
#
# make_directory("images/stacked/with_indices")
# make_directory("images/stacked/with_s1_seg")
# make_directory("images/stacked/all_19bands")
#
# make_directory("segmentation")
# make_directory("composites")
# make_directory("composites/10m")
# make_directory("composites/20m")
#
# make_directory("composites/10m_full")
# make_directory("composites/20m_full")
general_functions.make_all_dirs(working_dir)
# Step 1: Download S2 3imagery for the timescale
if not skip_download_and_preprocess:
config = configparser.ConfigParser()
config.read(path_to_config)
images_to_download = query.check_for_s2_data_by_date(
path_to_aoi, start_date, end_date, config, cloud_cover)
if not use_sen2cor:
images_to_download = query.filter_non_matching_s2_data(
images_to_download)
else:
images_to_download = query.filter_to_l1_data(images_to_download)
query.download_s2_data(images_to_download, "images/L1", "images/L2")
# Step 2: Preprocess S2 imagery. Perform atmospheric correction if needed, stack and mask 10 and 20m bands.
if use_sen2cor:
ras.atmospheric_correction("images/L1"
"images/L2",
sen2cor_path=sen2cor_path)
ras.preprocess_sen2_images("images/L2",
"images/merged/10m",
"images/L1",
cloud_threshold=0,
epsg=epsg_for_map,
bands=("B02", "B03", "B04", "B08"),
out_resolution=10)
ras.preprocess_sen2_images("images/L2",
"images/merged/20m",
"images/L1",
cloud_threshold=0,
epsg=epsg_for_map,
bands=("B02", "B03", "B04", "B05", "B06",
"B07", "B8A", "B11", "B12"),
out_resolution=20)
if not skip_composite:
# Step 2.5: Build a pair of cloud-free composites
sort_into_tile("images/merged/10m")
sort_into_tile("images/merged/20m")
for tile in os.listdir("images/merged/10m"):
tile_path = os.path.join("images/merged/10m", tile)
this_composite_path = ras.composite_directory(
tile_path, "composites/10m")
new_composite_path = "{}_{}.tif".format(
this_composite_path.rsplit('.')[0], tile)
os.rename(this_composite_path, new_composite_path)
for tile in os.listdir("images/merged/20m"):
tile_path = os.path.join("images/merged/20m", tile)
this_composite_path = ras.composite_directory(
tile_path, "composites/20m")
new_composite_path = "{}_{}.tif".format(
this_composite_path.rsplit('.')[0], tile)
os.rename(this_composite_path, new_composite_path)
# Step 3: Generate the bands. Time for the New Bit.
clip_to_aoi = False
if clip_to_aoi:
for image in os.listdir("composites/10m_full"):
if image.endswith(".tif"):
image_path_10m_full = os.path.join("composites/10m_full",
image)
image_path_20m_full = os.path.join("composites/20m_full",
image)
image_path_10m_clipped = os.path.join("composites/10m", image)
image_path_20m_clipped = os.path.join("composites/20m", image)
# config = configparser.ConfigParser()
# conf = config.read(path_to_config)
# print(conf)
# aoi = config['cocoa_mapping']['path_to_aoi']
aoi = "/media/ubuntu/Data/Ghana/cocoa_big/shp/cocoa_big.shp"
ras.clip_raster(raster_path=image_path_10m_full,
aoi_path=aoi,
out_path=image_path_10m_clipped,
srs_id=32630)
ras.clip_raster(raster_path=image_path_20m_full,
aoi_path=aoi,
out_path=image_path_20m_clipped,
srs_id=32630)
do_segmentation = True
if do_segmentation == True:
for image in os.listdir("composites/10m"):
if image.endswith(".tif"):
with TemporaryDirectory() as td:
image_path_10m = os.path.join("composites/10m", image)
image_path_20m = os.path.join("composites/20m", image)
resample_path_20m_v1 = os.path.join(
td,
image) # You'll probably regret this later, roberts.
shutil.copy(image_path_20m, resample_path_20m_v1)
ras.resample_image_in_place(resample_path_20m_v1, 10)
index_image_path = os.path.join(td, "index_image.tif")
temp_pre_seg_path = os.path.join(td, "pre_seg.tif")
temp_seg_path = os.path.join(td, "seg.tif")
temp_shp_path = os.path.join(td, "outline.shp")
temp_clipped_seg_path = os.path.join(td, "seg_clip.tif")
# This bit's your show, Qing
temp_s1_outline_path = os.path.join(td, "s1_outline.shp")
ras.get_extent_as_shp(in_ras_path=image_path_10m,
out_shp_path=temp_s1_outline_path)
resample_path_20m = os.path.join(
td, image[:-4] + '_to_10moutline.tif')
general_functions.clip_rst(
in_tif=resample_path_20m_v1,
outline_shp=temp_s1_outline_path,
out_tif=resample_path_20m,
keep_rst_extent=False)
generate_veg_index_tif(image_path_10m, resample_path_20m,
index_image_path)
ras.stack_images([index_image_path, image_path_10m],
"images/stacked/with_indices/" + image)
# Now, we do Highly Experimental Image Segmentation. Please put on your goggles.
# SAGA, please.
# Meatball science time
vis_10m = gdal.Open(image_path_10m)
vis_20m_resampled = gdal.Open(resample_path_20m)
vis_10m_array = vis_10m.GetVirtualMemArray()
vis_20m_array = vis_20m_resampled.GetVirtualMemArray()
# NIR, SWIR, red
array_to_classify = np.stack([
vis_10m_array[3, ...], vis_20m_array[7, ...],
vis_10m_array[2, ...]
])
g, arr = general_functions.read_tif(intif=image_path_10m)
general_functions.create_tif(filename=temp_pre_seg_path,
g=g,
Nx=arr.shape[1],
Ny=arr.shape[2],
new_array=array_to_classify,
noData=0,
data_type=gdal.GDT_UInt32)
out_segment_tif = os.path.join("segmentation", image)
segment_image(temp_pre_seg_path, out_segment_tif)
print('Generate brighness raster from the segments')
make_directory("segmentation/brightness")
out_brightness_value_ras = os.path.join(
"segmentation/brightness", image)
output_filtered_value_ras = False
ras.get_extent_as_shp(in_ras_path=temp_pre_seg_path,
out_shp_path=temp_shp_path)
general_functions.clip_rst(in_tif=out_segment_tif,
outline_shp=temp_shp_path,
out_tif=temp_clipped_seg_path,
keep_rst_extent=False)
cal_seg_mean(
temp_pre_seg_path,
temp_clipped_seg_path,
out_brightness_value_ras,
output_filtered_value_ras=output_filtered_value_ras)
# image_20m_6bands_array = vis_20m_array[3:,:,:]
# try:
# os.mkdir("composites/20m/20m_6bands")
# except FileExistsError:
# pass
#
# out_20m_tif_for_stack = os.path.join("composites/20m/20m_6bands", image)
# general_functions.create_tif(filename=out_20m_tif_for_stack,g=g,Nx=arr.shape[1],Ny=arr.shape[2],
# new_array=image_20m_6bands_array,data_type=gdal.GDT_UInt16,noData=0)
do_stack = True
if do_stack == True:
# Step 4: Stack the new bands with the S1, seg, and 6 band 20m rasters
for image in os.listdir("images/stacked/with_indices"):
if image.endswith(".tif"):
path_to_image = os.path.join("images/stacked/with_indices",
image)
path_to_brightness_image = os.path.join(
"segmentation/brightness", image)
# path_to_20m_image = os.path.join("composites/20m/20m_6bands", image)
# ras.stack_images([path_to_image, path_to_s1_image,path_to_20m_image,out_brightness_value_ras], os.path.join("images/stacked/all_19bands", image))
ras.stack_images([
path_to_image, path_to_s1_image, path_to_brightness_image
], os.path.join("images/stacked/with_s1_seg", image))
#sys.exit()
#
# Step 5: Classify with trained model
for image in os.listdir("images/stacked/with_s1_seg"):
if image.endswith(".tif"):
path_to_image = os.path.join("images/stacked/with_s1_seg", image)
path_to_out = os.path.join("output", image)
PYEO_model.classify_image(path_to_image, path_to_model,
path_to_out)
# s2_veg_s1_bands[16,:,:] = vh
# s2_veg_s1_bands[17,:,:] = vv
#
# pdb.set_trace()
#veg_index_tif = veg_array_out
#s2_10m_tif = intif_10m
#vh_int_tif = s1_vh_testsite_tif
#vv_int_tif = s1_vv_testsite_tif
#s2simp_veg_s1_bands_out = intif_10m[:-4] + '_vegIndex_s1.tif'
# os.system('gdal_merge.py -separate -ot Int16 -o ' + s2simp_veg_s1_bands_out + ' '
# + veg_index_tif + ' ' + s2_10m_tif + ' ' + vh_int_tif + ' ' + vv_int_tif)
s2simp_veg_s1_bands = "/media/ubuntu/storage/Ghana/cocoa_upscale_test/s2/s2_20180219_testsite_vegIndex_s1.tif"
#scale to 255
s2simp_veg_s1_bands_255 = s2simp_veg_s1_bands[:-4] +'_255.tif'
#training_image_255 = scale_to_255(intif=s2simp_veg_s1_bands,outtif= s2simp_veg_s1_bands_255)
#4. classify image
out_model_path = "/media/ubuntu/storage/Ghana/cocoa_upscale_test/ghana_gridsearch.pkl"
image_path = "/media/ubuntu/storage/Ghana/cocoa_upscale_test/s2/s2_20180219_testsite_vegIndex_s1.tif"
model = joblib.load(out_model_path)
out_image = image_path[:-4] + '_classified.tif'
PYEO_model.classify_image(in_image_path=image_path, model=model, out_image_path=out_image, num_chunks=20)