def process_shoreline(contours, georef, image_epsg, settings):
"""
Converts the contours from image coordinates to world coordinates. This function also removes
the contours that are too small to be a shoreline (based on the parameter
settings['min_length_sl'])
KV WRL 2018
Arguments:
-----------
contours: np.array or list of np.array
image contours as detected by the function find_contours
georef: np.array
vector of 6 elements [Xtr, Xscale, Xshear, Ytr, Yshear, Yscale]
image_epsg: int
spatial reference system of the image from which the contours were extracted
settings: dict
contains the following fields:
output_epsg: int
output spatial reference system
min_length_sl: float
minimum length of shoreline perimeter to be kept (in meters)
Returns:
-----------
shoreline: np.array
array of points with the X and Y coordinates of the shoreline
"""
# convert pixel coordinates to world coordinates
contours_world = SDS_tools.convert_pix2world(contours, georef)
# convert world coordinates to desired spatial reference system
contours_epsg = SDS_tools.convert_epsg(contours_world, image_epsg,
settings['output_epsg'])
# remove contours that have a perimeter < min_length_sl (provided in settings dict)
# this enables to remove the very small contours that do not correspond to the shoreline
contours_long = []
for l, wl in enumerate(contours_epsg):
coords = [(wl[k, 0], wl[k, 1]) for k in range(len(wl))]
a = LineString(coords) # shapely LineString structure
if a.length >= settings['min_length_sl']:
contours_long.append(wl)
# format points into np.array
x_points = np.array([])
y_points = np.array([])
for k in range(len(contours_long)):
x_points = np.append(x_points, contours_long[k][:, 0])
y_points = np.append(y_points, contours_long[k][:, 1])
contours_array = np.transpose(np.array([x_points, y_points]))
shoreline = contours_array
return shoreline
def process_shoreline(contours, cloud_mask, georef, image_epsg, settings):
"""
Converts the contours from image coordinates to world coordinates. This function also removes
the contours that are too small to be a shoreline (based on the parameter
settings['min_length_sl'])
KV WRL 2018
Arguments:
-----------
contours: np.array or list of np.array
image contours as detected by the function find_contours
cloud_mask: np.array
2D cloud mask with True where cloud pixels are
georef: np.array
vector of 6 elements [Xtr, Xscale, Xshear, Ytr, Yshear, Yscale]
image_epsg: int
spatial reference system of the image from which the contours were extracted
settings: dict
contains the following fields:
output_epsg: int
output spatial reference system
min_length_sl: float
minimum length of shoreline perimeter to be kept (in meters)
Returns:
-----------
shoreline: np.array
array of points with the X and Y coordinates of the shoreline
"""
# convert pixel coordinates to world coordinates
contours_world = SDS_tools.convert_pix2world(contours, georef)
# convert world coordinates to desired spatial reference system
contours_epsg = SDS_tools.convert_epsg(contours_world, image_epsg,
settings['output_epsg'])
# remove contours that have a perimeter < min_length_sl (provided in settings dict)
# this enables to remove the very small contours that do not correspond to the shoreline
contours_long = []
for l, wl in enumerate(contours_epsg):
coords = [(wl[k, 0], wl[k, 1]) for k in range(len(wl))]
a = LineString(coords) # shapely LineString structure
if a.length >= settings['min_length_sl']:
contours_long.append(wl)
# format points into np.array
x_points = np.array([])
y_points = np.array([])
for k in range(len(contours_long)):
x_points = np.append(x_points, contours_long[k][:, 0])
y_points = np.append(y_points, contours_long[k][:, 1])
contours_array = np.transpose(np.array([x_points, y_points]))
shoreline = contours_array
# now remove any shoreline points that are attached to cloud pixels
if sum(sum(cloud_mask)) > 0:
# get the coordinates of the cloud pixels
idx_cloud = np.where(cloud_mask)
idx_cloud = np.array([(idx_cloud[0][k], idx_cloud[1][k])
for k in range(len(idx_cloud[0]))])
# convert to world coordinates and same epsg as the shoreline points
coords_cloud = SDS_tools.convert_epsg(
SDS_tools.convert_pix2world(idx_cloud, georef), image_epsg,
settings['output_epsg'])[:, :-1]
# only keep the shoreline points that are at least 30m from any cloud pixel
idx_keep = np.ones(len(shoreline)).astype(bool)
for k in range(len(shoreline)):
if np.any(
np.linalg.norm(shoreline[k, :] -
coords_cloud, axis=1) < 30):
idx_keep[k] = False
shoreline = shoreline[idx_keep]
return shoreline
def get_reference_sl(metadata, settings):
"""
Allows the user to manually digitize a reference shoreline that is used seed
the shoreline detection algorithm. The reference shoreline helps to detect
the outliers, making the shoreline detection more robust.
KV WRL 2018
Arguments:
-----------
metadata: dict
contains all the information about the satellite images that were downloaded
settings: dict with the following keys
'inputs': dict
input parameters (sitename, filepath, polygon, dates, sat_list)
'cloud_thresh': float
value between 0 and 1 indicating the maximum cloud fraction in
the cropped image that is accepted
'cloud_mask_issue': boolean
True if there is an issue with the cloud mask and sand pixels
are erroneously being masked on the images
'output_epsg': int
output spatial reference system as EPSG code
Returns:
-----------
reference_shoreline: np.array
coordinates of the reference shoreline that was manually digitized.
This is also saved as a .pkl and .geojson file.
"""
sitename = settings['inputs']['sitename']
filepath_data = settings['inputs']['filepath']
pts_coords = []
# check if reference shoreline already exists in the corresponding folder
filepath = os.path.join(filepath_data, sitename)
filename = sitename + '_reference_shoreline.pkl'
# if it exist, load it and return it
if filename in os.listdir(filepath):
print('Reference shoreline already exists and was loaded')
with open(os.path.join(filepath, sitename + '_reference_shoreline.pkl'), 'rb') as f:
refsl = pickle.load(f)
return refsl
# otherwise get the user to manually digitise a shoreline on S2, L8 or L5 images (no L7 because of scan line error)
else:
# first try to use S2 images (10m res for manually digitizing the reference shoreline)
if 'S2' in metadata.keys():
satname = 'S2'
filepath = SDS_tools.get_filepath(settings['inputs'],satname)
filenames = metadata[satname]['filenames']
# if no S2 images, try L8 (15m res in the RGB with pansharpening)
elif not 'S2' in metadata.keys() and 'L8' in metadata.keys():
satname = 'L8'
filepath = SDS_tools.get_filepath(settings['inputs'],satname)
filenames = metadata[satname]['filenames']
# if no S2 images and no L8, use L5 images (L7 images have black diagonal bands making it
# hard to manually digitize a shoreline)
elif not 'S2' in metadata.keys() and not 'L8' in metadata.keys() and 'L5' in metadata.keys():
satname = 'L5'
filepath = SDS_tools.get_filepath(settings['inputs'],satname)
filenames = metadata[satname]['filenames']
else:
raise Exception('You cannot digitize the shoreline on L7 images (because of gaps in the images), add another L8, S2 or L5 to your dataset.')
# create figure
fig, ax = plt.subplots(1,1, figsize=[18,9], tight_layout=True)
mng = plt.get_current_fig_manager()
mng.window.showMaximized()
# loop trhough the images
for i in range(len(filenames)):
# read image
fn = SDS_tools.get_filenames(filenames[i],filepath, satname)
im_ms, georef, cloud_mask, im_extra, im_QA, im_nodata = preprocess_single(fn, satname, settings['cloud_mask_issue'])
# calculate cloud cover
cloud_cover = np.divide(sum(sum(cloud_mask.astype(int))),
(cloud_mask.shape[0]*cloud_mask.shape[1]))
# skip image if cloud cover is above threshold
if cloud_cover > settings['cloud_thresh']:
continue
# rescale image intensity for display purposes
im_RGB = rescale_image_intensity(im_ms[:,:,[2,1,0]], cloud_mask, 99.9)
# plot the image RGB on a figure
ax.axis('off')
ax.imshow(im_RGB)
# decide if the image if good enough for digitizing the shoreline
ax.set_title('Press <right arrow> if image is clear enough to digitize the shoreline.\n' +
'If the image is cloudy press <left arrow> to get another image', fontsize=14)
# set a key event to accept/reject the detections (see https://stackoverflow.com/a/15033071)
# this variable needs to be immuatable so we can access it after the keypress event
skip_image = False
key_event = {}
def press(event):
# store what key was pressed in the dictionary
key_event['pressed'] = event.key
# let the user press a key, right arrow to keep the image, left arrow to skip it
# to break the loop the user can press 'escape'
while True:
btn_keep = plt.text(1.1, 0.9, 'keep ⇨', size=12, ha="right", va="top",
transform=ax.transAxes,
bbox=dict(boxstyle="square", ec='k',fc='w'))
btn_skip = plt.text(-0.1, 0.9, '⇦ skip', size=12, ha="left", va="top",
transform=ax.transAxes,
bbox=dict(boxstyle="square", ec='k',fc='w'))
btn_esc = plt.text(0.5, 0, '<esc> to quit', size=12, ha="center", va="top",
transform=ax.transAxes,
bbox=dict(boxstyle="square", ec='k',fc='w'))
plt.draw()
fig.canvas.mpl_connect('key_press_event', press)
plt.waitforbuttonpress()
# after button is pressed, remove the buttons
btn_skip.remove()
btn_keep.remove()
btn_esc.remove()
# keep/skip image according to the pressed key, 'escape' to break the loop
if key_event.get('pressed') == 'right':
skip_image = False
break
elif key_event.get('pressed') == 'left':
skip_image = True
break
elif key_event.get('pressed') == 'escape':
plt.close()
raise StopIteration('User cancelled checking shoreline detection')
else:
plt.waitforbuttonpress()
if skip_image:
ax.clear()
continue
else:
# create two new buttons
add_button = plt.text(0, 0.9, 'add', size=16, ha="left", va="top",
transform=plt.gca().transAxes,
bbox=dict(boxstyle="square", ec='k',fc='w'))
end_button = plt.text(1, 0.9, 'end', size=16, ha="right", va="top",
transform=plt.gca().transAxes,
bbox=dict(boxstyle="square", ec='k',fc='w'))
# add multiple reference shorelines (until user clicks on <end> button)
pts_sl = np.expand_dims(np.array([np.nan, np.nan]),axis=0)
geoms = []
while 1:
add_button.set_visible(False)
end_button.set_visible(False)
# update title (instructions)
ax.set_title('Click points along the shoreline (enough points to capture the beach curvature).\n' +
'Start at one end of the beach.\n' + 'When finished digitizing, click <ENTER>',
fontsize=14)
plt.draw()
# let user click on the shoreline
pts = ginput(n=50000, timeout=1e9, show_clicks=True)
pts_pix = np.array(pts)
# convert pixel coordinates to world coordinates
pts_world = SDS_tools.convert_pix2world(pts_pix[:,[1,0]], georef)
# interpolate between points clicked by the user (1m resolution)
pts_world_interp = np.expand_dims(np.array([np.nan, np.nan]),axis=0)
for k in range(len(pts_world)-1):
pt_dist = np.linalg.norm(pts_world[k,:]-pts_world[k+1,:])
xvals = np.arange(0,pt_dist)
yvals = np.zeros(len(xvals))
pt_coords = np.zeros((len(xvals),2))
pt_coords[:,0] = xvals
pt_coords[:,1] = yvals
phi = 0
deltax = pts_world[k+1,0] - pts_world[k,0]
deltay = pts_world[k+1,1] - pts_world[k,1]
phi = np.pi/2 - np.math.atan2(deltax, deltay)
tf = transform.EuclideanTransform(rotation=phi, translation=pts_world[k,:])
pts_world_interp = np.append(pts_world_interp,tf(pt_coords), axis=0)
pts_world_interp = np.delete(pts_world_interp,0,axis=0)
# save as geometry (to create .geojson file later)
geoms.append(geometry.LineString(pts_world_interp))
# convert to pixel coordinates and plot
pts_pix_interp = SDS_tools.convert_world2pix(pts_world_interp, georef)
pts_sl = np.append(pts_sl, pts_world_interp, axis=0)
ax.plot(pts_pix_interp[:,0], pts_pix_interp[:,1], 'r--')
ax.plot(pts_pix_interp[0,0], pts_pix_interp[0,1],'ko')
ax.plot(pts_pix_interp[-1,0], pts_pix_interp[-1,1],'ko')
# update title and buttons
add_button.set_visible(True)
end_button.set_visible(True)
ax.set_title('click on <add> to digitize another shoreline or on <end> to finish and save the shoreline(s)',
fontsize=14)
plt.draw()
# let the user click again (<add> another shoreline or <end>)
pt_input = ginput(n=1, timeout=1e9, show_clicks=False)
pt_input = np.array(pt_input)
# if user clicks on <end>, save the points and break the loop
if pt_input[0][0] > im_ms.shape[1]/2:
add_button.set_visible(False)
end_button.set_visible(False)
plt.title('Reference shoreline saved as ' + sitename + '_reference_shoreline.pkl and ' + sitename + '_reference_shoreline.geojson')
plt.draw()
ginput(n=1, timeout=3, show_clicks=False)
plt.close()
break
pts_sl = np.delete(pts_sl,0,axis=0)
# convert world image coordinates to user-defined coordinate system
image_epsg = metadata[satname]['epsg'][i]
pts_coords = SDS_tools.convert_epsg(pts_sl, image_epsg, settings['output_epsg'])
# save the reference shoreline as .pkl
filepath = os.path.join(filepath_data, sitename)
with open(os.path.join(filepath, sitename + '_reference_shoreline.pkl'), 'wb') as f:
pickle.dump(pts_coords, f)
# also store as .geojson in case user wants to drag-and-drop on GIS for verification
for k,line in enumerate(geoms):
gdf = gpd.GeoDataFrame(geometry=gpd.GeoSeries(line))
gdf.index = [k]
gdf.loc[k,'name'] = 'reference shoreline ' + str(k+1)
# store into geodataframe
if k == 0:
gdf_all = gdf
else:
gdf_all = gdf_all.append(gdf)
gdf_all.crs = {'init':'epsg:'+str(image_epsg)}
# convert from image_epsg to user-defined coordinate system
gdf_all = gdf_all.to_crs({'init': 'epsg:'+str(settings['output_epsg'])})
# save as geojson
gdf_all.to_file(os.path.join(filepath, sitename + '_reference_shoreline.geojson'),
driver='GeoJSON', encoding='utf-8')
print('Reference shoreline has been saved in ' + filepath)
break
# check if a shoreline was digitised
if len(pts_coords) == 0:
raise Exception('No cloud free images are available to digitise the reference shoreline,'+
'download more images and try again')
return pts_coords
def get_reference_sl(metadata, settings):
"""
Allows the user to manually digitize a reference shoreline that is used seed the shoreline
detection algorithm. The reference shoreline helps to detect the outliers, making the shoreline
detection more robust.
KV WRL 2018
Arguments:
-----------
metadata: dict
contains all the information about the satellite images that were downloaded
settings: dict
contains the following fields:
'cloud_thresh': float
value between 0 and 1 indicating the maximum cloud fraction in the image that is accepted
'sitename': string
name of the site (also name of the folder where the images are stored)
'output_epsg': int
epsg code of the desired spatial reference system
Returns:
-----------
reference_shoreline: np.array
coordinates of the reference shoreline that was manually digitized
"""
sitename = settings['inputs']['sitename']
filepath_data = settings['inputs']['filepath']
# check if reference shoreline already exists in the corresponding folder
filepath = os.path.join(filepath_data, sitename)
filename = sitename + '_reference_shoreline.pkl'
if filename in os.listdir(filepath):
print('Reference shoreline already exists and was loaded')
with open(
os.path.join(filepath, sitename + '_reference_shoreline.pkl'),
'rb') as f:
refsl = pickle.load(f)
return refsl
else:
# first try to use S2 images (10m res for manually digitizing the reference shoreline)
if 'S2' in metadata.keys():
satname = 'S2'
filepath = SDS_tools.get_filepath(settings['inputs'], satname)
filenames = metadata[satname]['filenames']
# if no S2 images, try L8 (15m res in the RGB with pansharpening)
elif not 'S2' in metadata.keys() and 'L8' in metadata.keys():
satname = 'L8'
filepath = SDS_tools.get_filepath(settings['inputs'], satname)
filenames = metadata[satname]['filenames']
# if no S2 images and no L8, use L5 images (L7 images have black diagonal bands making it
# hard to manually digitize a shoreline)
elif not 'S2' in metadata.keys() and not 'L8' in metadata.keys(
) and 'L5' in metadata.keys():
satname = 'L5'
filepath = SDS_tools.get_filepath(settings['inputs'], satname)
filenames = metadata[satname]['filenames']
else:
raise Exception(
'You cannot digitize the shoreline on L7 images, add another L8, S2 or L5 to your dataset.'
)
# loop trhough the images
for i in range(len(filenames)):
# read image
fn = SDS_tools.get_filenames(filenames[i], filepath, satname)
im_ms, georef, cloud_mask, im_extra, imQA = preprocess_single(
fn, satname, settings['cloud_mask_issue'])
# calculate cloud cover
cloud_cover = np.divide(
sum(sum(cloud_mask.astype(int))),
(cloud_mask.shape[0] * cloud_mask.shape[1]))
# skip image if cloud cover is above threshold
if cloud_cover > settings['cloud_thresh']:
continue
# rescale image intensity for display purposes
im_RGB = rescale_image_intensity(im_ms[:, :, [2, 1, 0]],
cloud_mask, 99.9)
# plot the image RGB on a figure
fig = plt.figure()
fig.set_size_inches([18, 9])
fig.set_tight_layout(True)
plt.axis('off')
plt.imshow(im_RGB)
# decide if the image if good enough for digitizing the shoreline
plt.title(
'click <keep> if image is clear enough to digitize the shoreline.\n'
+ 'If not (too cloudy) click on <skip> to get another image',
fontsize=14)
keep_button = plt.text(0,
0.9,
'keep',
size=16,
ha="left",
va="top",
transform=plt.gca().transAxes,
bbox=dict(boxstyle="square", ec='k',
fc='w'))
skip_button = plt.text(1,
0.9,
'skip',
size=16,
ha="right",
va="top",
transform=plt.gca().transAxes,
bbox=dict(boxstyle="square", ec='k',
fc='w'))
mng = plt.get_current_fig_manager()
mng.window.showMaximized()
# let user click on the image once
pt_input = ginput(n=1, timeout=1e9, show_clicks=False)
pt_input = np.array(pt_input)
# if clicks next to <skip>, show another image
if pt_input[0][0] > im_ms.shape[1] / 2:
plt.close()
continue
else:
# remove keep and skip buttons
keep_button.set_visible(False)
skip_button.set_visible(False)
# create two new buttons
add_button = plt.text(0,
0.9,
'add',
size=16,
ha="left",
va="top",
transform=plt.gca().transAxes,
bbox=dict(boxstyle="square",
ec='k',
fc='w'))
end_button = plt.text(1,
0.9,
'end',
size=16,
ha="right",
va="top",
transform=plt.gca().transAxes,
bbox=dict(boxstyle="square",
ec='k',
fc='w'))
# add multiple reference shorelines (until user clicks on <end> button)
pts_sl = np.expand_dims(np.array([np.nan, np.nan]), axis=0)
geoms = []
while 1:
add_button.set_visible(False)
end_button.set_visible(False)
# update title (instructions)
plt.title(
'Click points along the shoreline (enough points to capture the beach curvature).\n'
+ 'Start at one end of the beach.\n' +
'When finished digitizing, click <ENTER>',
fontsize=14)
plt.draw()
# let user click on the shoreline
pts = ginput(n=50000, timeout=1e9, show_clicks=True)
pts_pix = np.array(pts)
# convert pixel coordinates to world coordinates
pts_world = SDS_tools.convert_pix2world(
pts_pix[:, [1, 0]], georef)
# interpolate between points clicked by the user (1m resolution)
pts_world_interp = np.expand_dims(np.array(
[np.nan, np.nan]),
axis=0)
for k in range(len(pts_world) - 1):
pt_dist = np.linalg.norm(pts_world[k, :] -
pts_world[k + 1, :])
xvals = np.arange(0, pt_dist)
yvals = np.zeros(len(xvals))
pt_coords = np.zeros((len(xvals), 2))
pt_coords[:, 0] = xvals
pt_coords[:, 1] = yvals
phi = 0
deltax = pts_world[k + 1, 0] - pts_world[k, 0]
deltay = pts_world[k + 1, 1] - pts_world[k, 1]
phi = np.pi / 2 - np.math.atan2(deltax, deltay)
tf = transform.EuclideanTransform(
rotation=phi, translation=pts_world[k, :])
pts_world_interp = np.append(pts_world_interp,
tf(pt_coords),
axis=0)
pts_world_interp = np.delete(pts_world_interp, 0, axis=0)
# save as geometry (to create .geojson file later)
geoms.append(geometry.LineString(pts_world_interp))
# convert to pixel coordinates and plot
pts_pix_interp = SDS_tools.convert_world2pix(
pts_world_interp, georef)
pts_sl = np.append(pts_sl, pts_world_interp, axis=0)
plt.plot(pts_pix_interp[:, 0], pts_pix_interp[:, 1], 'r--')
plt.plot(pts_pix_interp[0, 0], pts_pix_interp[0, 1], 'ko')
plt.plot(pts_pix_interp[-1, 0], pts_pix_interp[-1, 1],
'ko')
# update title and buttons
add_button.set_visible(True)
end_button.set_visible(True)
plt.title(
'click <add> to digitize another shoreline or <end> to finish and save the shoreline(s)',
fontsize=14)
plt.draw()
# let the user click again (<add> another shoreline or <end>)
pt_input = ginput(n=1, timeout=1e9, show_clicks=False)
pt_input = np.array(pt_input)
# if user clicks on <end>, save the points and break the loop
if pt_input[0][0] > im_ms.shape[1] / 2:
add_button.set_visible(False)
end_button.set_visible(False)
plt.title('Reference shoreline saved as ' + sitename +
'_reference_shoreline.pkl and ' + sitename +
'_reference_shoreline.geojson')
plt.draw()
ginput(n=1, timeout=3, show_clicks=False)
plt.close()
break
pts_sl = np.delete(pts_sl, 0, axis=0)
# convert world image coordinates to user-defined coordinate system
image_epsg = metadata[satname]['epsg'][i]
pts_coords = SDS_tools.convert_epsg(pts_sl, image_epsg,
settings['output_epsg'])
# save the reference shoreline as .pkl
filepath = os.path.join(filepath_data, sitename)
with open(
os.path.join(filepath,
sitename + '_reference_shoreline.pkl'),
'wb') as f:
pickle.dump(pts_coords, f)
# also store as .geojson in case user wants to drag-and-drop on GIS for verification
for k, line in enumerate(geoms):
gdf = gpd.GeoDataFrame(geometry=gpd.GeoSeries(line))
gdf.index = [k]
gdf.loc[k, 'name'] = 'reference shoreline ' + str(k + 1)
# store into geodataframe
if k == 0:
gdf_all = gdf
else:
gdf_all = gdf_all.append(gdf)
gdf_all.crs = {'init': 'epsg:' + str(image_epsg)}
# convert from image_epsg to user-defined coordinate system
gdf_all = gdf_all.to_crs(
{'init': 'epsg:' + str(settings['output_epsg'])})
# save as geojson
gdf_all.to_file(os.path.join(
filepath, sitename + '_reference_shoreline.geojson'),
driver='GeoJSON',
encoding='utf-8')
print('Reference shoreline has been saved in ' + filepath)
break
return pts_coords