def crop2common(im1, im2, im1out, im2out):
"""
Crop two images to the common extent.
Inputs:
im1: str, path to the first image
im2: str, path to the second image
im1out: str, path to the cropped first image
im2out: str, path to the cropped second image
"""
#Read first image extent
img1 = raster.SingleBandRaster(im1, load_data=False)
xmin1, xmax1, ymin1, ymax1 = img1.extent
#Read first image extent
img2 = raster.SingleBandRaster(im2, load_data=False)
xmin2, xmax2, ymin2, ymax2 = img2.extent
# compute common extent
xmin = max(xmin1, xmin2)
xmax = min(xmax1, xmax2)
ymin = max(ymin1, ymin2)
ymax = min(ymax1, ymax2)
#Crop first image
cmd = "gdalwarp -te %.8f %.8f %.8f %.8f %s %s -overwrite" % (
xmin, ymin, xmax, ymax, im1, im1out)
print cmd
os.system(cmd)
#Crop second image
cmd = "gdalwarp -te %.8f %.8f %.8f %.8f %s %s -overwrite" % (
xmin, ymin, xmax, ymax, im2, im2out)
print cmd
os.system(cmd)
def plot_background(self, bg_file, region='all', coarse=False):
"""
Plot a background image onto the Basemap, in black and white.
Optionally, coarsen resolution of background image, which will
result in smaller file sizes in saved vector formats.
Inputs:
bg_file : str, path and filename of single-band GeoTIFF
region : 'all' or latlon tuple (lonll,lonur,latll,latur)
coarse : False or int to coarsen image by (e.g. 2)
"""
if region == 'all':
bg = georaster.SingleBandRaster(bg_file)
else:
bg = georaster.SingleBandRaster(bg_file,
load_data=region,
latlon=True)
# Reduce image resolution
if coarse != False:
if type(coarse) == int:
bg.r = bg.r[::coarse, ::coarse]
bg.r = np.where(bg.r == 0, np.nan, bg.r) #remove black color
plt.imshow(bg.r,
cmap=cm.Greys_r,
extent=bg.get_extent_projected(self.map),
interpolation='nearest')
def plotRegion(occupancyRasterFile, occupancyGrid, plotsfile=None, width=10):
occ_raster = gdal.Open(occupancyRasterFile)
cols = occ_raster.RasterXSize
rows = occ_raster.RasterYSize
height = (float(rows) / float(cols)) * float(width)
f = plt.figure(figsize=(width, height))
ax = f.add_subplot(111)
region = georaster.SingleBandRaster(occ_raster, load_data=False)
minx, maxx, miny, maxy = region.extent
m = Basemap(projection='cyl',
llcrnrlon=minx - .005,
llcrnrlat=miny - .005,
urcrnrlon=maxx + .005,
urcrnrlat=maxy + .005,
resolution='h')
image = georaster.SingleBandRaster(occ_raster,
load_data=(minx, maxx, miny, maxy),
latlon=True)
ax.imshow(image.r, extent=(0, cols, 0, rows), zorder=10, alpha=0.6)
if plotsfile is not None:
plt.savefig(plotsfile + "_region.eps",
bbox_inches="tight",
format="eps",
dpi=300)
return ax
def predict_on_file(model, fname, crop=True, flips=flip.FLIP_FULL, channels=4):
"""Predicts on 4 separate regions of the input file, then pieces them together if crop=True
Averages predictions on flips of each image based upon flip parameter
Chooses which input tensor to create depending upon number of channels
channels == 4 -- RGB+DSM
channels == 3 -- RGB
channels == 1 -- DSM"""
original_img = load_img(fname) / 255.0
dsm = georaster.SingleBandRaster(fname.replace('RGB', 'DSM')).r
dtm = georaster.SingleBandRaster(fname.replace('RGB', 'DTM')).r
original_dem = (dsm - dtm) / 9.0
pred_array = []
if crop:
for img, dem_img in zip(get_crops(original_img),
get_crops(original_dem)):
out = _mini_predict_on_file(model,
img,
dem_img,
channels=channels,
flips=flips)
pred_array.append(
np.transpose((out[0] * 255).astype(np.uint8),
(1, 2, 0)).squeeze())
return reconstruct(*pred_array)
else:
out = _mini_predict_on_file(model,
original_img,
original_dem,
flips=flips,
channels=channels)
return np.transpose((out[0] * 255).astype(np.uint8),
(1, 2, 0)).squeeze()
def extract_gimp_extents(L0dir,filestart,save_to):
"""
Extract geographic extents of GIMP tiles and save them to CSV file.
L0dir : str, location of GIMP data,
e.g. '/disk/scratch/local.2/L0data/GIMP/'
filestart : str, e.g. GimpIceMask_15m_tile
save_to : str, e.g. Gimp_15m_tile_GIMP.csv
"""
rows = np.arange(0,6)
cols = np.arange(0,6)
with open('GimpIceMask_15m_extents.csv','w') as f_out:
f_out.write('row,col,xmin,xmax,ymin,ymax\n')
for r in rows:
for c in cols:
print r,c
f = filestart + str(r) + '_' + str(c) + '.tif'
im = georaster.SingleBandRaster(L0dir + f)
ex = im.extent
f_out.write(str(r) + ',' + str(c) + ',' + str(ex[0]) + ',' + \
str(ex[1]) + ',' + str(ex[2]) + ',' +str(ex[3]) + '\n')
f_out.close()
def create_latlon_da(geotiff_fn,
x_name,
y_name,
x_coords,
y_coords,
proj_info,
encoding='default'):
gtiff = georaster.SingleBandRaster(geotiff_fn, load_data=False)
lon, lat = gtiff.coordinates(latlon=True)
gtiff = None
coords_geo = {y_name: y_coords, x_name: x_coords}
if encoding == 'default':
encoding = {
'_FillValue': -9999.,
'dtype': 'int16',
'scale_factor': 0.000000001
}
lon_array = xr.DataArray(lon, coords=coords_geo, dims=['y', 'x'])
lon_array.attrs['grid_mapping'] = proj_info.attrs['grid_mapping_name']
lon_array.attrs['units'] = 'degrees'
lon_array.attrs['standard_name'] = 'longitude'
lat_array = xr.DataArray(lat, coords=coords_geo, dims=['y', 'x'])
lat_array.attrs['grid_mapping'] = proj_info.attrs['grid_mapping_name']
lat_array.attrs['units'] = 'degrees'
lat_array.attrs['standard_name'] = 'latitude'
return (lon_array, lat_array)
def plot_mask(self, mask_file, color='turquoise', region='all', alpha=1):
"""
Plot masked values of the provided dataset.
This can be useful to display 'bad' regions. If the colormap is set to
discrete, the regions will be plotted in red, otherwise in white.
Arguments:
mask_file : str, path to geoTIFF of mask
color : any matplotlib color, str or RGB triplet
region : optional, latlon tuple (lonll,lonur,latll,latur)
"""
if region == 'all':
mask = georaster.SingleBandRaster(mask_file)
else:
mask = georaster.SingleBandRaster(mask_file,
load_data=region,
latlon=True)
#Pixels outside mask are transparent
mask.r = np.where(mask.r == 0, np.nan, 1)
# Now plot the bad data values
cmap = cm.jet
if color == 'turquoise':
cmap.set_over((64. / 255, 224. / 255, 208. / 255))
elif color == 'discr':
cmap.set_under(
(155. / 255, 0. / 255, 0. / 255)) #gaps displayed in red
else:
try:
cmap.set_over(eval(color)) #color is a RGB triplet
except NameError:
cmap.set_over(color) #color is str, e.g red, white...
plt.imshow(mask.r,
extent=mask.get_extent_projected(self.map),
cmap=cmap,
vmin=-1,
vmax=0,
interpolation='nearest',
alpha=alpha)
class TestAttributes(unittest.TestCase): im = georaster.SingleBandRaster(os.path.join(georaster.test_data_path, 'LE71400412000304SGS00_B4_crop.TIF')) def test_extent(self): left, right, bottom, top = self.im.extent self.assertAlmostEqual(left , 478000.000, places=3) self.assertAlmostEqual(right , 502000.000, places=3) self.assertAlmostEqual(bottom, 3088490.000, places=3) self.assertAlmostEqual(top , 3108140.000, places=3)
def plot_dem(self, dem_file, region='all', azdeg=100, altdeg=65):
"""
Plot a DEM using light-shading on the Basemap.
Inputs:
dem_file : path and filename of GeoTIFF DEM.
region : 'all' or latlon tuple (lonll,lonur,latll,latur)
azdeg/altdeg : azimuth (measured clockwise from south) and altitude (measured up from the plane of the surface) of the light source in degrees.
"""
if region == 'all':
dem = georaster.SingleBandRaster(dem_file)
else:
dem = georaster.SingleBandRaster(dem_file,
load_data=region,
latlon=True)
ls = LightSource(azdeg=azdeg, altdeg=altdeg)
rgb = ls.shade(dem.r, cmap=cm.Greys_r)
plt.imshow(rgb,
extent=dem.get_extent_projected(self.map),
interpolation='nearest')
class TestCoordinateTransforms(unittest.TestCase):
"""
Decimal degrees values calculated from Corner Coordinates output by gdalinfo,
converted from DMS to decimal using:
https://www.rapidtables.com/convert/number/degrees-minutes-seconds-to-degrees.html
"""
im = georaster.SingleBandRaster(os.path.join(georaster.test_data_path, 'LE71400412000304SGS00_B4_crop.TIF'))
def test_get_extent_latlon(self):
left, right, bottom, top = self.im.get_extent_latlon()
self.assertAlmostEqual(left , 86.7760428, places=2)
self.assertAlmostEqual(right , 87.0203598, places=2)
self.assertAlmostEqual(bottom, 27.9211689, places=2)
self.assertAlmostEqual(top , 28.098735, places=2)
def test_get_extent_projected(self):
# Use example of web mercator projection
test_proj = pyproj.Proj('+init=epsg:3785')
left, right, bottom, top = self.im.get_extent_projected(test_proj)
# self.assertAlmostEqual(left , 9659864.900, places=2)
# self.assertAlmostEqual(right , 9687063.081, places=2)
# self.assertAlmostEqual(bottom, 3239029.409, places=2)
# self.assertAlmostEqual(top , 3261427.912, places=2)
self.assertAlmostEqual(left , 9659905.707276639, places=2)
self.assertAlmostEqual(right , 9687062.141583452, places=2)
self.assertAlmostEqual(bottom, 3239038.6219950984, places=2)
self.assertAlmostEqual(top , 3261427.72683907, places=2)
def test_coord_to_px_latlon(self):
return
def test_coord_to_px_projected(self):
return
def test_coord_to_px_projected_cellCorner(self):
return
def test_coordinates(self):
return
class TestValueRetrieval(unittest.TestCase):
im = georaster.SingleBandRaster(
os.path.join(georaster.test_data_path,
'LE71400412000304SGS00_B4_crop.TIF'))
def test_value_at_coords_latlon(self):
# 1. Lat/Lon, cell center coordinate.
val = self.im.value_at_coords(86.90271, 28.00108, latlon=True)
assert val == 67
def test_value_at_coords_projected(self):
# 2. Projected, cell center coordinate.
val = self.im.value_at_coords(490434.605, 3097325.642)
assert val == 67
def construct_label_map(self, selected_big_tile):
print("=========== CONSTRUCTING LABEL MAP ===========")
label_map = np.zeros((self.input_dimension,self.input_dimension))
listing = os.listdir(self.label_dir)
listing.sort()
i=1
for filename in listing[1:]:
if(not(selected_big_tile in filename)):
continue
path = self.label_dir+'/'+filename
print(path)
if(os.stat(path).st_size != 0):
im = georaster.SingleBandRaster(path)
mask = im.r!=0.0
label_map *= (~mask) #Used so that overlapping pixels dont get summed value.
mask = mask.astype(int)*i
label_map += mask
i+=1
print("Total number of classes in labels is: "+str(i)+" And S2_preprocessor has: "+str(self.nb_classes))
return(label_map)
def WorldPopSeries(admin_level,
sf,
file,
data_dir,
check_pickles=True,
save=True,
interpolate=[]):
"""Function to create a series of pop estimates at admin level based on the
world pop estimate.
check_pickles: bool, telling the function to check the data dir for premade versions
to save time (since computing them takes a while).
save: save a pickle if you compute something new."""
## Based on the naming convention, this is the pickle to
## check (and save to)
i = file.find("PAK")
pickle = data_dir + "BirthsWorldPop\\_processed\\admin" + str(
admin_level) + "_births_" + file[i:i + 7] + ".pkl"
if check_pickles:
try:
births = pd.read_pickle(pickle)
compute = False
except:
compute = True
else:
compute = True
## If we get here, we have to compute it.
if compute:
print("No pickle matching this request (" + pickle +
")! Starting computation...")
## Get the tif as a georaster
print("Getting the georaster...")
image = georaster.SingleBandRaster(data_dir + file)
## Remove zero pixels
mask = image.r > 0.
## Get GPS and births, then delete the
## georaster to free up memory for the
## data frame.
print("Removing empty pixels...")
X, Y = image.coordinates()
births = image.r[mask]
X = X[mask]
Y = Y[mask]
del image, mask
## Create a data frame with admin level column.
print("Creating a data frame...")
df = np.array([np.arange(len(X), dtype=int), X, Y, births]).T
df = pd.DataFrame(df, columns=["pt", "X", "Y", "births"])
del X, Y, births
print("Assigning admin" + str(admin_level) + " labels...")
df = AdminLevelFromGPS(df,
sf,
admin_level,
col_cluster="pt",
col_x="X",
col_y="Y",
output_col="admin")
## Compute the births, and release the
## data frame.
print("Computing total population...")
births = df.groupby("admin")["births"].sum()
del df
## Save this run (before interpolation, so you can experiment with
## different interpolation methods if needed).
if save:
print("Saving as " + pickle)
births.to_pickle(pickle)
## Interpolate missing parts of the tiff if needed, done regardless
## of the pickle incase you want to overwrite nan's in different ways
## and compare.
names = NamesFromSf(sf, admin_level)
if interpolate and (len(births.index) != len(names)):
print("Interpolating missing admin units...")
## Get centers
admin_units = AdminLevelSubset(sf, admin_level)
centers = {n: ShapeCenter(s) for n, s in admin_units.items()}
centers = pd.DataFrame(centers.values(),
index=pd.Index(centers.keys(),
name="admin_level"),
columns=["center_x", "center_y"])
## Merge the data frames to force nan's at
## missing admin units
births = pd.concat([centers, births], axis=1)
data = births.dropna()
## Now interpolate
points = data[["center_x", "center_y"]]
z = data[["births"]]
for method in interpolate:
births[method] = griddata(points,
z,
births[["center_x", "center_y"]],
method=method)
births["births"].fillna(births[method], inplace=True)
births = births["births"]
return births
missing_values = config.get('Params', 'missing_value').split(',')
missing_values = [int_or_float(b.strip()) for b in missing_values]
scale_factors = config.get('Params', 'scale_factor').split(',')
scale_factors = [int_or_float(b.strip()) for b in scale_factors]
# Move to output directory
os.chdir(io_path)
## Spatial coordinates
# Load up a geotiff to retrieve grid
infile = product + '.' + 'A' + str(year_start) + str(st).zfill(
3) + '.' + version + '.' + bands[0] + '.' + out_label + '.tif'
im = georaster.SingleBandRaster(infile)
nx = im.nx
ny = im.ny
x, y = im.coordinates()
# Coordinates returned by georaster are centre-of-pixel, set to lowerleft
x = x[1, :]
x = x - (im.xres / 2)
y = y[:, 1]
# Reverse y coordinate order - !!important!! otherwise slicing doesn't work properly
y = y[::-1]
y = y + (im.yres / 2)
# Iterate by year
for yr in range(year_start, year_end):
print(yr)
# sys.exit()
## Generate output DEM ##
print("\n*** Generate final DEM ***")
# Interpolation and output type hard-coded for now.
# ArcticDEM tiles exactly touch and are on a continuous grid, so no interpolation should be needed with the -tap option if spacing is kept the same
cmd = 'gdalwarp -r bilinear -ot Int16 --optfile %s %s ' % (list_file,
args.outfile)
if args.te is not None:
if args.latlon == True:
# reproject extent to DEM extent
img = raster.SingleBandRaster(dem_files[0], load_data=False)
x1, y1 = img.proj(lonmin, latmin)
x2, y2 = img.proj(lonmin, latmax)
x3, y3 = img.proj(lonmax, latmax)
x4, y4 = img.proj(lonmax, latmin)
xmin = min(x1, x2, x3, x4)
xmax = max(x1, x2, x3, x4)
ymin = min(y1, y2, y3, y4)
ymax = max(y1, y2, y3, y4)
else:
xmin, ymin, xmax, ymax = args.te
# -tap option ensures that the output grid is the same as the input grid for a same spacing (i.e. not shift is created during resampling)
cmd += ' -te %f %f %f %f -tap' % (xmin, ymin, xmax, ymax)
if args.tr is not None:
import georaster
import matplotlib.pyplot as plt
from mpl_toolkits.basemap import Basemap
import sys
fig = plt.figure(figsize=(8,8))
# full path to the geotiff file
fpath = sys.argv[1]
# read extent of image without loading
# good for values in degrees lat/long
# geotiff may use other coordinates and projection
my_image = georaster.SingleBandRaster(fpath, load_data=False)
# grab limits of image's extent
minx, maxx, miny, maxy = my_image.extent
# set Basemap with slightly larger extents
# set resolution at intermediate level "i"
m = Basemap( projection='cyl', \
llcrnrlon=minx-2, \
llcrnrlat=miny-2, \
urcrnrlon=maxx+2, \
urcrnrlat=maxy+2, \
resolution='i')
m.drawcoastlines(color="gray")
# m.fillcontinents(color='beige')
# load the geotiff image, assign it a variable
# =============================================================================
if len(sys.argv) < 4:
Usage()
sys.exit(1)
#Read arguments
im_ref = sys.argv[1]
im2crop = sys.argv[2]
outfile = sys.argv[3]
#Read reference image spatial reference system
ds = gdal.Open(im_ref)
srs_dest = osr.SpatialReference()
wkt = ds.GetProjectionRef()
srs_dest.ImportFromWkt(wkt)
proj = srs_dest.ExportToProj4()
#Read reference image size and corners coordinates
img = geo.SingleBandRaster(im_ref)
pixelWidth, pixelHeight = img.get_pixel_size()
xmin, xmax, ymin, ymax = img.extent
#Crop and reproject
cmd = "gdalwarp -te %.8f %.8f %.8f %.8f -tr %.8f %.8f -t_srs '%s' %s %s -overwrite" % (
xmin, ymin, xmax, ymax, pixelWidth, pixelHeight, proj, im2crop, outfile)
print cmd
os.system(cmd)
def create_gimp_raster(image_path,gimp_extents_csv,outfile,clean=False,
verbose=True,tile_type='dem'):
""" Create a raster from the GIMP Ice Mask/DEM corresponding to the
dimensions and extent of the input image.
Parameters:
image_path : str, path to a geo-referenced image (GeoTIFF)
gimp_extents_csv : str, path to csv containing extents of every 15 m
GIMP tile (generated by geoutils/extract_gimp_extent),
all mask files must be in same folder as this csv.
outfile : str, path to the output GeoTiff file
clean : True/False, default True, clean up after merging operation.
verbose: True/False, default False, display coordinate conversions
tile_type : str, 'dem' or 'mask' (ice mask) or 'ocean' (ocean mask)
Returns:
Creates the outfile.
AJT May 2014.
"""
# Name of each GIMP file before row/col notation
if tile_type == 'dem':
GIMP_FILE_PREFIX = GIMP_dem_prefix
elif tile_type == 'mask':
GIMP_FILE_PREFIX = GIMP_ice_prefix
elif tile_type == 'ocean':
GIMP_FILE_PREFIX = GIMP_ocean_prefix
# Name of temporary file to merge to
GIMP_TMP_MERGE = 'temporary_merge.tif'
# Base of GIMP dir
GIMP_DIR = '/'.join(gimp_extents_csv.split('/')[0:-1]) + '/'
# Import list of GIMP file extents
extents = pd.read_csv(gimp_extents_csv,delimiter=',',
header=0)
# Load image ref
in_image = georaster.SingleBandRaster(image_path,load_data=False)
# The landsat tiles and GIMP tiles are in different projection systems, so
# we must convert them both to lat/lon for comparison.
# Deal with landsat georeferencing
ex = in_image.extent
lxmin,lymin = in_image.proj(ex[0],ex[2],inverse=True)
lxmax,lymax = in_image.proj(ex[1],ex[3],inverse=True)
if verbose == True:
print 'Tile:'
print lxmin,lymin
print lxmax,lymax
# Set up GIMP tile projection (they are all the same)
proj_gimp = pyproj.Proj(GIMP_PROJ4)
# Find the GIMP tiles which cover area for this landsat tile
use = []
for ix,tile in extents.iterrows():
mxmin,mymin = proj_gimp(tile['xmin'],tile['ymin'],inverse=True)
mxmax,mymax = proj_gimp(tile['xmax'],tile['ymax'],inverse=True)
if verbose == True:
print tile['row'],tile['col']
print mxmin,mymin
print mxmax,mymax
xuse = False
yuse = False
if (lxmin > mxmin) and (lxmin < mxmax):
xuse = True
if (lxmax > mxmin) and (lxmax < mxmax):
xuse = True
if xuse == True:
if (lymin > mymin) and (lymin < mymax):
yuse = True
if (lymax > mymin) and (lymax < mymax):
yuse = True
if yuse == True:
use.append((int(tile['row']),int(tile['col'])))
if verbose == True:
print use
# Create string of GIMP files to build merged tile from
build_from = ''
for r,c in use:
build_from = build_from + GIMP_DIR + GIMP_FILE_PREFIX + str(r) + \
'_' + str(c) + '.tif '
# Do the merge
command = 'gdal_merge.py -ot "Int16" -o ' + GIMP_DIR + GIMP_TMP_MERGE + \
' ' + build_from
print "**** COMMAND : %s" %command; os.system(command)
# Create the merged GIMP tile for the input img at appropriate resolution
xres, yres = in_image.get_pixel_size()
command = "gdalwarp -t_srs '" + in_image.srs.ExportToProj4() + "' -te " + str(ex[0]-15000) + " " + \
str(ex[2]) + " " + str(ex[1]) + \
" " + str(ex[3]) + " -tr " + str(xres) + " " + str(yres) +\
" " + GIMP_DIR + GIMP_TMP_MERGE + " " + outfile
print "**** COMMAND : %s" %command; os.system(command)
# Clean up
if clean == True:
os.remove(GIMP_DIR + GIMP_TMP_MERGE)
# Number of positive finds needed in above window length for day to count as
# onset date. If you set the same as the window length then this many
# consecutive days are required. If you set it to less then the days may be
# non-consecutive. For values < window_bare then the day is only set as the
# onset day if n_dark + n_bad_quality_obs == window_bare, i.e. no bare
# ice/snow values will be allowed to occur in this time...do I want to allow
# a tolerance of e.g. + 0.02?
n_bare = 3
n_dark = 3
""" END PARAMETERS """
"""
Load mask
"""
# Mask generated by GDAL from 90 m ice sheet mask
mask = georaster.SingleBandRaster(
'/scratch/MOD09GA.006.SW/GIMP_IceMask_MAR613mSW.tif')
# Set off-ice-sheet areas to nan
mask.r = np.where(mask.r > 0, 1, np.nan)
mask.r[0:250, 0:300] = np.nan
mask.r[575:945, 0:157] = np.nan
mask.r[1169:1417, 0:150] = np.nan
mask.r[823:893, 150:200] = np.nan
mask.r[830:860, 197:214] = np.nan
"""
Compute basic stats, some of these are also computed year-by-year below
"""
if compute_basic == True:
modis_ds = xr.open_mfdataset('/scratch/MOD09GA.006.SW/*2017*b1234q.nc',
chunks={'TIME': 365})
b01 = modis_ds.sur_refl_b01_1
from matplotlib import rcParams
import pandas as pd
from matplotlib.collections import PatchCollection
from shapely.geometry import Point, Polygon, MultiPoint, MultiPolygon, LineString
from descartes import PolygonPatch
import georaster
import plotmap
rcParams['font.sans-serif'] = 'Arial'
rcParams['font.size'] = 6
rcParams['mathtext.fontset'] = 'stixsans'
err_thresh = 60.
mask = georaster.SingleBandRaster(
'/home/s1144267/rds/landsat/WRS12_annual/merge_1985_1986_snr4_rad340_nmin1_region_v3_pstere.mask.TIF'
)
dem = georaster.SingleBandRaster(
'/home/s1144267/rds/landsat/WRS12_annual/GIMP_dem_WRS12_merge_240m_pstere.TIF'
)
mask_vel = np.where((dem.r >= 400) & (dem.r <= 1100) & (mask.r == 1), 1, 0)
ntot = np.sum(mask_vel)
region = (-51.1, -49.2, 67.450714, 69.2)
lon_0 = -45
# Ice area
ice_mapo = plotmap.Map(extent=region, lon_0=lon_0)
shp_info = ice_mapo.map.readshapefile('Gimp_Ice_Mask_240m_EPSG4319',
'ice',
def coreg_with_IceSAT(args):
"""
Coregistration with the use of IceSAT data
"""
is_files = args.master_dem
## Read slave DEM ##
slave_dem = DEMRaster(args.slave_dem)
slave_dem.r = np.float32(slave_dem.r)
if args.nodata2 != 'none':
nodata = float(args.nodata2)
else:
band = slave_dem.ds.GetRasterBand(1)
nodata = band.GetNoDataValue()
# save original data for later resampling
dem2coreg = slave_dem
dem2coreg_save = np.copy(dem2coreg.r)
# compute DEM extent
lonmin, lonmax, latmin, latmax = dem2coreg.get_extent_latlon()
lonmin += 360
lonmax += 360
RoI = ((lonmin, latmin), (lonmin, latmax), (lonmax, latmax),
(lonmax, latmin), (lonmin, latmin))
## mask points ##
mask = raster.SingleBandRaster(args.maskfile)
if dem2coreg.r.shape != mask.r.shape:
print("Reproject mask")
mask = mask.reproject(dem2coreg.srs,
dem2coreg.nx,
dem2coreg.ny,
dem2coreg.extent[0],
dem2coreg.extent[3],
dem2coreg.xres,
dem2coreg.yres,
dtype=6,
nodata=nodata,
interp_type=1,
progress=True)
dem2coreg.r[mask.r > 0] = np.nan
## read Icesat data with DEM extent ##
all_lons, all_lats, all_elev = read_icesat_elev(is_files, RoI)
## compare slave DEM and Icesat ##
slave_elev = dem2coreg.interp(all_lons, all_lats, latlon=True)
dh = all_elev - slave_elev
dh[slave_elev == 0] = np.nan
xx, yy = dem2coreg.proj(all_lons, all_lats)
if args.plot == True:
pl.title('Icesat - slave DEM elev')
rgb = dem2coreg.shaded_relief(downsampl=5)
pl.scatter(xx, yy, c=dh, edgecolor='none', vmin=-20, vmax=20)
cb = pl.colorbar()
cb.set_label('Elevation difference (m)')
pl.show()
## compute slave DEM slope at Icesat points ##
print("Compute slope and aspect")
g2, g1 = np.gradient(dem2coreg.r)
distx = np.abs(dem2coreg.xres)
disty = np.abs(dem2coreg.yres)
slope_pix = np.sqrt((g1 / distx)**2 + (g2 / disty)**2)
aspect = np.arctan2(-g1, g2)
aspect = aspect + np.pi
slope_ds = raster.simple_write_geotiff('none',
slope_pix,
dem2coreg.ds.GetGeoTransform(),
wkt=dem2coreg.srs.ExportToWkt(),
dtype=6)
slope_raster = raster.SingleBandRaster(slope_ds)
slope_at_IS = slope_raster.interp(all_lons, all_lats, latlon=True)
aspect_ds = raster.simple_write_geotiff('none',
aspect,
dem2coreg.ds.GetGeoTransform(),
wkt=dem2coreg.srs.ExportToWkt(),
dtype=6)
aspect_raster = raster.SingleBandRaster(aspect_ds)
aspect_at_IS = aspect_raster.interp(all_lons, all_lats, latlon=True)
# slave DEM grid
xgrid = np.arange(dem2coreg.nx)
ygrid = np.arange(dem2coreg.ny)
X, Y = dem2coreg.coordinates()
## Print out some statistics
median = np.median(dh[np.isfinite(dh)])
NMAD_old = 1.4826 * np.median(np.abs(dh[np.isfinite(dh)] - median))
print("Statistics on initial dh")
print("Median : %f, NMAD : %f" % (median, NMAD_old))
## Iterations to estimate DEMs shift
print("Iteratively estimate DEMs shift")
slave_elev = dem2coreg.interp(all_lons, all_lats, latlon=True)
dh = all_elev - slave_elev
dh[slave_elev == 0] = np.nan
xoff, yoff = 0, 0
for i in range(args.niter):
# compute aspect/dh relationship
east, north, c = horizontal_shift(dh,
slope_at_IS,
aspect_at_IS,
plot=args.plot,
min_count=args.min_count)
print("#%i - Offset in pixels : (%f,%f)" % (i + 1, east, north))
xoff += east
yoff += north
#Update elevation difference
slave_elev = dem2coreg.interp(xx + xoff, yy + yoff)
dh = all_elev - slave_elev
dh[slave_elev == 0] = np.nan
# print some statistics
median = np.median(dh[np.isfinite(dh)])
NMAD_new = 1.4826 * np.median(np.abs(dh[np.isfinite(dh)] - median))
print("Median : %.2f, NMAD = %.2f, Gain : %.2f%%" %
(median, NMAD_new, (NMAD_new - NMAD_old) / NMAD_old * 100))
NMAD_old = NMAD_new
print("Final Offset in pixels (east, north) : (%f,%f)" % (xoff, yoff))
if args.save == True:
fname, ext = os.path.splitext(args.outfile)
fname += '_shift.txt'
f = open(fname, 'w')
f.write("Final Offset in pixels (east, north) : (%f,%f)" %
(xoff, yoff))
f.write("Final NMAD : %f" % NMAD_new)
f.close()
print("Offset saved in %s" % fname)
### Deramping ###
print("Deramping")
# remove points above altitude threshold (snow covered areas)
#if args.zmax!='none':
# dh[master_dem.r>int(args.zmax)] = np.nan
# remove points below altitude threshold (e.g sea ice)
zmin = 40
if zmin != 'none':
dh[slave_elev < int(zmin)] = np.nan
# remove points with slope higher than 20° that are more error-prone
# slope, aspect = dem2coreg.compute_slope()
# dh[slope>=20*np.pi/180] = np.nan
# dh[np.isnan(slope)] = np.nan
# remove outliers
med = np.median(dh[np.isfinite(dh)])
mad = 1.4826 * np.median(np.abs(dh[np.isfinite(dh)] - med))
dh[np.abs(dh - med) > 3 * mad] = np.nan
# estimate a ramp and remove it
ramp = deramping(dh, xx, yy, d=args.degree, plot=False)
# compute stats of deramped dh
tmp = dh - ramp(X, Y)
median = np.median(tmp)
NMAD_new = 1.4826 * np.median(np.abs(tmp[np.isfinite(tmp)] - median))
if args.save == True:
fname, ext = os.path.splitext(args.outfile)
fname += '_shift.txt'
f = open(fname, 'a')
f.write("Median after deramping : (%f)" % (median))
f.write("NMAD after deramping : %f" % NMAD_new)
f.close()
print("Post-deramping stats saved in %s" % fname)
# save to output file
if args.save == True:
fname, ext = os.path.splitext(args.outfile)
fname += '_ramp.TIF'
#fname = WD+'/ramp.out'
raster.simple_write_geotiff(fname,
ramp(X, Y),
dem2coreg.ds.GetGeoTransform(),
wkt=dem2coreg.srs.ExportToWkt(),
dtype=gdal.GDT_Float32)
#ramp(X,Y).tofile(fname)
print("Ramp saved in %s" % fname)
if args.plot == True:
pl.figure('ramp')
pl.scatter(xx, yy, c=ramp(xx, yy), edgecolor='none')
pl.colorbar()
pl.figure('before')
pl.imshow(rgb, extent=dem2coreg.extent, interpolation='bilinear')
pl.scatter(xx, yy, c=dh, edgecolor='none', vmin=-10, vmax=10)
pl.colorbar()
pl.figure('after')
pl.imshow(rgb, extent=dem2coreg.extent, interpolation='bilinear')
pl.scatter(xx,
yy,
c=dh - ramp(xx, yy),
edgecolor='none',
vmin=-10,
vmax=10)
pl.colorbar()
pl.show()
### Interpolate the slave DEM to the new grid ###
print("Interpolate DEM to new grid")
# fill NaN values for interpolation
nanval = np.isnan(dem2coreg_save)
slave_filled = np.where(np.isnan(dem2coreg_save), -9999, dem2coreg_save)
# Create spline function
f = RectBivariateSpline(ygrid, xgrid, slave_filled, kx=1, ky=1)
f2 = RectBivariateSpline(ygrid, xgrid, nanval, kx=1, ky=1)
# resample slave DEM in the new grid
znew = f(ygrid - yoff, xgrid + xoff) #postive y shift moves south
nanval_new = f2(ygrid - yoff, xgrid + xoff)
#remove filled values that have been interpolated
znew[nanval_new != 0] = np.nan
# update DEM
dem2coreg_save = znew
### Remove ramp ###
dem2coreg_save -= ramp(X, Y)
### Save to output file ###
raster.simple_write_geotiff(args.outfile,
dem2coreg_save,
dem2coreg.ds.GetGeoTransform(),
wkt=dem2coreg.srs.ExportToWkt(),
dtype=gdal.GDT_Float32)
for fname in fnames:
## Extract the date
date = pd.to_datetime(fname[fname.find("npp_") + 4:fname.find("-")],
format="%Y%m%d")
for sr in sf.shapeRecords():
## Get the dotname
dot_name = sr.record[1].lower()
## Get the associated part of the raster
bbox = sr.shape.bbox
extent = (bbox[0], bbox[2], bbox[1], bbox[3])
raster = georaster.SingleBandRaster(root + fname,
load_data=extent,
latlon=True)
## Compute total brightness and size of the
## bounding box
num_pixels = int(np.prod(raster.r.shape))
total_brightness = raster.r.sum()
## Append the result
entry = (dot_name, date, total_brightness, num_pixels)
brightness.append(entry)
if plot and dot_name == "asia:pakistan:punjab:lahore":
plt.imshow(raster.r, extent=raster.extent)
points = np.array(sr.shape.points)
plt.plot(points[:, 0], points[:, 1], color="k", lw=2)
def ClassifyImages(clf, img_path, img_stub, area_label, savefigs=False):
with xr.open_dataset(savefig_path + "S2vals.nc",
chunks={
'x': 2000,
'y': 2000
}) as S2vals:
# Set index for reducing data
band_idx = pd.Index([1, 2, 3, 4, 5, 6, 7, 8, 9], name='bands')
# concatenate the bands into a single dimension ('bands_idx') in the data array
concat = xr.concat([
S2vals.B02, S2vals.B03, S2vals.B04, S2vals.B05, S2vals.B06,
S2vals.B07, S2vals.B08, S2vals.B11, S2vals.B12
], band_idx)
# stack the values into a 1D array
stacked = concat.stack(allpoints=['y', 'x'])
# Transpose and rename so that DataArray has exactly the same layout/labels as the training DataArray.
# mask out nan areas not masked out by GIMP
stackedT = stacked.T
stackedT = stackedT.rename({'allpoints': 'samples'})
# apply classifier
predicted = clf.predict(stackedT)
# Unstack back to x,y grid
predicted = predicted.unstack(dim='samples')
#calculate albeod using Liang et al (2002) equation
albedo = xr.DataArray(0.356 * (concat.values[1]) + 0.13 * (concat.values[3]) + 0.373 * \
(concat.values[6]) + 0.085 * (concat.values[7]) + 0.072 * (concat.values[8]) - 0.0018)
#update mask so that both GIMP mask and areas not sampled by S2 but not masked by GIMP both = 0
mask2 = (S2vals.Icemask.values == 1) & (
concat.sum(dim='bands') > 0) & (S2vals.Cloudmask.values == 0)
# collate predicted map, albedo map and projection info into xarray dataset
# 1) Retrieve projection info from S2 datafile and add to netcdf
srs = osr.SpatialReference()
srs.ImportFromProj4('+init=epsg:32622') # Get info for UTM zone 22N
proj_info = xr.DataArray(0, encoding={'dtype': np.dtype('int8')})
proj_info.attrs['projected_crs_name'] = srs.GetAttrValue('projcs')
proj_info.attrs['grid_mapping_name'] = 'UTM'
proj_info.attrs['scale_factor_at_central_origin'] = srs.GetProjParm(
'scale_factor')
proj_info.attrs['standard_parallel'] = srs.GetProjParm(
'latitude_of_origin')
proj_info.attrs[
'straight_vertical_longitude_from_pole'] = srs.GetProjParm(
'central_meridian')
proj_info.attrs['false_easting'] = srs.GetProjParm('false_easting')
proj_info.attrs['false_northing'] = srs.GetProjParm('false_northing')
proj_info.attrs['latitude_of_projection_origin'] = srs.GetProjParm(
'latitude_of_origin')
# 2) Create associated lat/lon coordinates DataArrays using georaster (imports geo metadata without loading img)
# see georaster docs at https:/media.readthedocs.org/pdf/georaster/latest/georaster.pdf
S2 = georaster.SingleBandRaster(img_path + img_stub + 'B02_20m.jp2',
load_data=False)
lon, lat = S2.coordinates(latlon=True)
S2 = None
S2 = xr.open_rasterio(img_path + img_stub + 'B02_20m.jp2',
chunks={
'x': 2000,
'y': 2000
})
coords_geo = {'y': S2['y'], 'x': S2['x']}
S2 = None
lon_array = xr.DataArray(lon,
coords=coords_geo,
dims=['y', 'x'],
encoding={
'_FillValue': -9999.,
'dtype': 'int16',
'scale_factor': 0.000000001
})
lon_array.attrs['grid_mapping'] = 'UTM'
lon_array.attrs['units'] = 'degrees'
lon_array.attrs['standard_name'] = 'longitude'
lat_array = xr.DataArray(lat,
coords=coords_geo,
dims=['y', 'x'],
encoding={
'_FillValue': -9999.,
'dtype': 'int16',
'scale_factor': 0.000000001
})
lat_array.attrs['grid_mapping'] = 'UTM'
lat_array.attrs['units'] = 'degrees'
lat_array.attrs['standard_name'] = 'latitude'
# 3) add predicted map array and add metadata
predictedxr = xr.DataArray(predicted.values,
coords=coords_geo,
dims=['y', 'x'])
predictedxr = predictedxr.fillna(0)
predictedxr = predictedxr.where(mask2 > 0)
predictedxr.encoding = {
'dtype': 'int16',
'zlib': True,
'_FillValue': -9999
}
predictedxr.name = 'Surface Class'
predictedxr.attrs[
'long_name'] = 'Surface classified using Random Forest'
predictedxr.attrs['units'] = 'None'
predictedxr.attrs[
'key'] = 'Snow:1; Water:2; Cryoconite:3; Clean Ice:4; Light Algae:5; Heavy Algae:6'
predictedxr.attrs['grid_mapping'] = 'UTM'
# add albedo map array and add metadata
albedoxr = xr.DataArray(albedo.values,
coords=coords_geo,
dims=['y', 'x'])
albedoxr = albedoxr.fillna(0)
albedoxr = albedoxr.where(mask2 > 0)
albedoxr.encoding = {
'dtype': 'int16',
'scale_factor': 0,
'zlib': True,
'_FillValue': -9999
}
albedoxr.name = 'Surface albedo computed after Knap et al. (1999) narrowband-to-broadband conversion'
albedoxr.attrs['units'] = 'dimensionless'
albedoxr.attrs['grid_mapping'] = 'UTM'
# collate data arrays into a dataset
dataset = xr.Dataset({
'classified': (['x', 'y'], predictedxr),
'albedo': (['x', 'y'], albedoxr),
'mask': (['x', 'y'], S2vals.Icemask.values),
'Projection': proj_info,
'longitude': (['x', 'y'], lon_array),
'latitude': (['x', 'y'], lat_array)
})
# add metadata for dataset
dataset.attrs['Conventions'] = 'CF-1.4'
dataset.attrs['Author'] = 'Joseph Cook (University of Sheffield, UK)'
dataset.attrs[
'title'] = 'Classified surface and albedo maps produced from Sentinel-2 ' \
'imagery of the SW Greenland Ice Sheet'
# Additional geo-referencing
dataset.attrs['nx'] = len(dataset.x)
dataset.attrs['ny'] = len(dataset.y)
dataset.attrs['xmin'] = float(dataset.x.min())
dataset.attrs['ymax'] = float(dataset.y.max())
dataset.attrs['spacing'] = 20
# NC conventions metadata for dimensions variables
dataset.x.attrs['units'] = 'meters'
dataset.x.attrs['standard_name'] = 'projection_x_coordinate'
dataset.x.attrs['point_spacing'] = 'even'
dataset.x.attrs['axis'] = 'x'
dataset.y.attrs['units'] = 'meters'
dataset.y.attrs['standard_name'] = 'projection_y_coordinate'
dataset.y.attrs['point_spacing'] = 'even'
dataset.y.attrs['axis'] = 'y'
dataset.to_netcdf(
savefig_path +
"{}_Classification_and_Albedo_Data.nc".format(area_label),
mode='w')
dataset = None
if savefigs:
cmap1 = mpl.colors.ListedColormap([
'purple', 'white', 'royalblue', 'black', 'lightskyblue',
'mediumseagreen', 'darkgreen'
])
cmap1.set_under(color='white') # make sure background is white
cmap2 = plt.get_cmap('Greys_r') # reverse greyscale for albedo
cmap2.set_under(color='white') # make sure background is white
fig, axes = plt.subplots(figsize=(10, 8), ncols=1, nrows=2)
predictedxr.plot(ax=axes[0], cmap=cmap1, vmin=0, vmax=6)
plt.ylabel('Latitude (UTM Zone 22N)'), plt.xlabel(
'Longitude (UTM Zone 22N)')
plt.title(
'Greenland Ice Sheet from Sentinel 2 classified using Random Forest Classifier (top) and albedo (bottom)'
)
axes[0].grid(None)
axes[0].set_aspect('equal')
albedoxr.plot(ax=axes[1], cmap=cmap2, vmin=0, vmax=1)
plt.ylabel('Latitude (UTM Zone 22N)'), plt.xlabel(
'Longitude (UTM Zone 22N)')
axes[1].set_aspect('equal')
axes[1].grid(None)
fig.tight_layout()
plt.savefig(
str(savefig_path +
"{}_Sentinel_Classified_Albedo.png".format(area_label)),
dpi=300)
plt.close()
return
crs = xr.DataArray(0, encoding={'dtype': np.dtype('int8')})
crs.attrs['projected_crs_name'] = srs.GetAttrValue('projcs')
crs.attrs['grid_mapping_name'] = 'universal_transverse_mercator'
crs.attrs['scale_factor_at_central_origin'] = srs.GetProjParm(
'scale_factor')
crs.attrs['standard_parallel'] = srs.GetProjParm('latitude_of_origin')
crs.attrs['straight_vertical_longitude_from_pole'] = srs.GetProjParm(
'central_meridian')
crs.attrs['false_easting'] = srs.GetProjParm('false_easting')
crs.attrs['false_northing'] = srs.GetProjParm('false_northing')
crs.attrs['latitude_of_projection_origin'] = srs.GetProjParm(
'latitude_of_origin')
## Create associated lat/lon coordinates DataArrays
uav_gr = georaster.SingleBandRaster('NETCDF:"%s%s":Band1' %
(fn_path, flights[flight]),
load_data=False)
grid_lon, grid_lat = uav_gr.coordinates(latlon=True)
uav = xr.open_dataset(fn_path + flights[flight],
chunks={
'x': 1000,
'y': 1000
})
coords_geo = {'y': uav['y'], 'x': uav['x']}
uav = None
lon_da = xr.DataArray(grid_lon,
coords=coords_geo,
dims=['y', 'x'],
encoding={
'_FillValue': -9999.,
import georaster as gr
# data_nc = netCDF4.Dataset(r'Y:\tmp\kmu\meps\arome_arctic_pp_1km_latest.nc')
# la = data_nc.variables['land_area_fraction'][:]
#
# mask_nc = netCDF4.Dataset(r'N:\Prosjekter\APS\VarslinOmr2018Svalbard2.nc')
#
# m = mask_nc.variables['VarslingsOmr2018Land'][:]
#
# mask3003 = np.ma.masked_not_equal(m, 3003)
#
# la3003 = np.where(m==3003, 2, np.flipud(la))
# m3003 = np.where(m==3003, np.flipud(la), m)
# plt.imshow(m3003, vmin=3000, vmax= 3005)
# plt.show()
raster = r'N:\Prosjekter\APS\svalbard_regions\VarslingsOmr2018Land.tif'
rdata = gr.SingleBandRaster(raster)
#(xmin, xsize, x, ymax, y, ysize) = data.geot
# print(rdata.projection)
print(rdata.extent, rdata.srs)
print(rdata.srs.GetProjParm('central_meridian'))
print(rdata.ds)
plt.imshow(rdata.r, vmin=3000, vmax=3005)
plt.show()
### ...export to netCDF
def ClassifyImages(S2vals, clf, icemask, cloudmask, savefigs=False, save_netcdf = False):
# get dimensions of each band layer
lenx, leny = np.shape(S2vals[0])
# convert image bands into single 5-dimensional numpy array
S2valsT = S2vals.reshape(9, lenx * leny) # reshape into 5 x 1D arrays
S2valsT = S2valsT.transpose() # transpose so that bands are read as features
# create albedo array by applying Knap (1999) narrowband - broadband conversion
albedo_array = np.array([0.356 * (S2vals[0]) + 0.13 * (S2vals[2]) + 0.373 * (
S2vals[6]) + 0.085 * (S2vals[7]) + 0.072 * (S2vals[8]) - 0.0018])
# apply ML algorithm to 4-value array for each pixel - predict surface type
predicted = clf.predict(S2valsT)
predicted = np.array(predicted)
# convert surface class (string) to a numeric value for plotting
predicted[predicted == 'SN'] = float(1)
predicted[predicted == 'WAT'] = float(2)
predicted[predicted == 'CC'] = float(3)
predicted[predicted == 'CI'] = float(4)
predicted[predicted == 'LA'] = float(5)
predicted[predicted == 'HA'] = float(6)
predicted[predicted == 'Zero'] = float(0)
# ensure array data type is float (required for imshow)
predicted = predicted.astype(float)
# reshape 1D array back into original image dimensions
predicted = np.reshape(predicted, [lenx, leny])
albedo = np.reshape(albedo_array, [lenx, leny])
# apply cloud mask to ignore pixels obscured by cloud
predicted = np.ma.masked_where(cloudmask==1, predicted)
albedo = np.ma.masked_where(cloudmask==1, albedo)
# apply GIMP mask to ignore non-ice surfaces
predicted = np.ma.masked_where(icemask==0, predicted)
albedo = np.ma.masked_where(icemask==0, albedo)
# mask out areas not covered by Sentinel tile, but not excluded by GIMP mask
predicted = np.ma.masked_where(predicted <=0, predicted)
albedo = np.ma.masked_where(albedo <=0, albedo)
# collate predicted map, albedo map and projection info into xarray dataset
# 1) Retrieve projection info from uav datafile and add to netcdf
srs = osr.SpatialReference()
srs.ImportFromProj4('+init=epsg:32623')
proj_info = xr.DataArray(0, encoding={'dtype': np.dtype('int8')})
proj_info.attrs['projected_crs_name'] = srs.GetAttrValue('projcs')
proj_info.attrs['grid_mapping_name'] = 'UTM'
proj_info.attrs['scale_factor_at_central_origin'] = srs.GetProjParm('scale_factor')
proj_info.attrs['standard_parallel'] = srs.GetProjParm('latitude_of_origin')
proj_info.attrs['straight_vertical_longitude_from_pole'] = srs.GetProjParm('central_meridian')
proj_info.attrs['false_easting'] = srs.GetProjParm('false_easting')
proj_info.attrs['false_northing'] = srs.GetProjParm('false_northing')
proj_info.attrs['latitude_of_projection_origin'] = srs.GetProjParm('latitude_of_origin')
# 2) Create associated lat/lon coordinates DataArrays using georaster (imports geo metadata without loading img)
# see georaster docs at https: // media.readthedocs.org / pdf / georaster / latest / georaster.pdf
S2 = georaster.SingleBandRaster('NETCDF:"%s":Band1' % (str(img_path+'B02.nc')),
load_data=False)
lon, lat = S2.coordinates(latlon=True)
S2 = None # close file
S2 = xr.open_dataset((str(img_path+'B02.nc')), chunks={'x': 2000, 'y': 2000})
coords_geo = {'y': S2['y'], 'x': S2['x']}
S2 = None # close file
lon_array = xr.DataArray(lon, coords=coords_geo, dims=['y', 'x'],
encoding={'_FillValue': -9999., 'dtype': 'int16', 'scale_factor': 0.000000001})
lon_array.attrs['grid_mapping'] = 'UTM'
lon_array.attrs['units'] = 'degrees'
lon_array.attrs['standard_name'] = 'longitude'
lat_array = xr.DataArray(lat, coords=coords_geo, dims=['y', 'x'],
encoding={'_FillValue': -9999., 'dtype': 'int16', 'scale_factor': 0.000000001})
lat_array.attrs['grid_mapping'] = 'UTM'
lat_array.attrs['units'] = 'degrees'
lat_array.attrs['standard_name'] = 'latitude'
# 3) add predicted map array and add metadata
predictedxr = xr.DataArray(predicted, coords=coords_geo, dims=['y', 'x'])
predictedxr.encoding = {'dtype': 'int16', 'zlib': True, '_FillValue': -9999}
predictedxr.name = 'Surface Class'
predictedxr.attrs['long_name'] = 'Surface classified using Random Forest'
predictedxr.attrs['units'] = 'None'
predictedxr.attrs[
'key'] = 'Unknown:0; Snow:1; Water:2; Cryoconite:3; Clean Ice:4; Light Algae:5; Heavy Algae:6'
predictedxr.attrs['grid_mapping'] = 'UTM'
# add albedo map array and add metadata
albedoxr = xr.DataArray(albedo, coords=coords_geo, dims=['y', 'x'])
albedoxr.encoding = {'dtype': 'int16', 'scale_factor': 0, 'zlib': True, '_FillValue': -9999}
albedoxr.name = 'Surface albedo computed after Knap et al. (1999) narrowband-to-broadband conversion'
albedoxr.attrs['units'] = 'dimensionless'
albedoxr.attrs['grid_mapping'] = 'UTM'
# collate data arrays into a dataset
dataset = xr.Dataset({
'classified': (['x', 'y'], predictedxr),
'albedo': (['x', 'y'], albedoxr),
'Icemask': (['x','y'],icemask),
'Cloudmask': (['x', 'y'], cloudmask),
'Projection': proj_info,
'longitude': (['x', 'y'], lon_array),
'latitude': (['x', 'y'], lat_array)})
# add metadata for dataset
dataset.attrs['Conventions'] = 'CF-1.4'
dataset.attrs['Author'] = 'Joseph Cook (University of Sheffield, UK)'
dataset.attrs[
'title'] = 'Classified surface and albedo maps produced from Sentinel-2 ' \
'imagery of the SW Greenland Ice Sheet'
# Additional geo-referencing
dataset.attrs['nx'] = len(dataset.x)
dataset.attrs['ny'] = len(dataset.y)
dataset.attrs['xmin'] = float(dataset.x.min())
dataset.attrs['ymax'] = float(dataset.y.max())
dataset.attrs['spacing'] = 20
# NC conventions metadata for dimensions variables
dataset.x.attrs['units'] = 'meters'
dataset.x.attrs['standard_name'] = 'projection_x_coordinate'
dataset.x.attrs['point_spacing'] = 'even'
dataset.x.attrs['axis'] = 'x'
dataset.y.attrs['units'] = 'meters'
dataset.y.attrs['standard_name'] = 'projection_y_coordinate'
dataset.y.attrs['point_spacing'] = 'even'
dataset.y.attrs['axis'] = 'y'
# save dataset to netcdf if requested
if save_netcdf:
dataset.to_netcdf(savefig_path + "Classification_and_Albedo_Data.nc")
if savefigs:
cmap1 = mpl.colors.ListedColormap(
['purple', 'white', 'royalblue', 'black', 'lightskyblue', 'mediumseagreen', 'darkgreen'])
cmap1.set_under(color='white') # make sure background is white
cmap2 = plt.get_cmap('Greys_r') # reverse greyscale for albedo
cmap2.set_under(color='white') # make sure background is white
fig = plt.figure(figsize=(15, 30))
# first subplot = classified map
class_labels = ['Unknown', 'Snow', 'Water', 'Cryoconite', 'Clean Ice', 'Light Algae', 'Heavy Algae']
ax1 = plt.subplot(211)
img = plt.imshow(predicted, cmap=cmap1,vmin=0, vmax=7)
cbar = fig.colorbar(mappable = img, ax=ax1, fraction=0.045)
n_classes = len(class_labels)
tick_locs = np.arange(0.5,len(class_labels),1)
cbar.set_ticks(tick_locs)
cbar.ax.set_yticklabels(class_labels, fontsize=26, rotation=0, va='center')
cbar.set_label('Surface Class',fontsize=22)
plt.title("Classified Surface Map\nProjection: UTM Zone 23", fontsize = 30), ax1.set_aspect('equal')
plt.xticks([0, 2745, 5490],['-51.000235','-49.708602','-48.418656'],fontsize=26, rotation=45),plt.xlabel('Longitude (decimal degrees)',fontsize=26)
plt.yticks([0,2745, 5490],['67.615437','67.610307','67.594927'],fontsize=26),plt.ylabel('Latitude (decimal degrees)',fontsize=26)
plt.grid(None)
# second subplot = albedo map
ax2 = plt.subplot(212)
img2 = plt.imshow(albedo, cmap=cmap2, vmin=0, vmax=1)
cbar2 = plt.colorbar(mappable=img2, fraction=0.045)
cbar2.ax.set_yticklabels(labels=[0,0.2,0.4,0.6,0.8,1.0], fontsize=26)
cbar2.set_label('Albedo',fontsize=26)
plt.xticks([0, 2745, 5490],['-51.000235','-49.708602','-48.418656'],fontsize=26,rotation=45),plt.xlabel('Longitude (decimal degrees)',fontsize=26)
plt.yticks([0,2745, 5490],['67.615437','67.610307','67.594927'],fontsize=26),plt.ylabel('Latitude (decimal degrees)',fontsize=26)
plt.grid(None),plt.title("Albedo Map\nProjection: UTM Zone 23",fontsize=30)
ax2.set_aspect('equal')
plt.tight_layout()
plt.savefig(str(savefig_path + "Sentinel_Classified_Albedo.png"), dpi=300)
plt.close()
return predicted, albedo, dataset
import osgeo import numpy as np import pandas as pd import os # os.environ['PROJ_LIB'] = r'E:/Anaconda/pkgs/proj4-5.2.0-ha925a31_1/Library/share' import matplotlib.pyplot as plt import matplotlib.path as mplPath import shapefile import pyproj import georaster import pycrs import toolbar from mpl_toolkits.axes_grid1 import make_axes_locatable pathNS = 'd27m12y2016S014815' img = georaster.SingleBandRaster(pathNS + '\image.tif') sigNS, LaNS, LoNS, thetaNS = toolbar.readFolder(pathNS) extent = [135, 45, 165, 63] # extent = [ 138, 55, 148, 60 ] fig = plt.figure(figsize=(8, 6)) ax = fig.add_axes([0.1, 0.1, 0.8, 0.8]) # setup mercator map projection. m = toolbar.makeMap(extent) # draw parallels m.drawparallels(np.arange(45, 65, 2), labels=[1, 0, 0, 1], color='grey') # draw meridians m.drawmeridians(np.arange(135, 165, 2), labels=[1, 0, 0, 1], color='grey') plt.imshow(image.r, extent=image.extent)
def classify_images(clf, img_file, plot_maps = True, savefigs = False, save_netcdf = False):
startTime = datetime.now() # start timer
# open uav file using xarray. Use "with ... as ..." method so that file auto-closes after use
with xr.open_dataset(img_file) as uav:
# calibration against ASD Field Spec
uav['Band1'] -= 0.17
uav['Band2'] -= 0.18
uav['Band3'] -= 0.15
uav['Band4'] -= 0.16
uav['Band5'] -= 0.05
# Set index for reducing data
band_idx = pd.Index([1, 2, 3, 4, 5], name='bands')
# concatenate the bands into a single dimension ('bands_idx') in the data array
concat = xr.concat([uav.Band1, uav.Band2, uav.Band3, uav.Band4, uav.Band5], band_idx)
# Mask nodata areas
concat2 = concat.where(concat.sum(dim='bands') > 0)
# stack the values into a 1D array
stacked = concat2.stack(allpoints=['y', 'x'])
# Transpose and rename so that DataArray has exactly the same layout/labels as the training DataArray.
stackedT = stacked.T
stackedT = stackedT.rename({'allpoints': 'samples'})
stackedT = stackedT.where(stackedT.sum(dim='bands') > 0).dropna(dim='samples')
# apply classifier (make use of all cores)
predicted_temp = clf.predict(stackedT)
# Unstack back to x,y grid and save as numpy array
predicted = np.array(predicted_temp.unstack(dim='samples'))
# convert albedo array to numpy array for analysis
# obtain albedo by aplying Knap (1999) narrowband to broadband albedo conversion.
concat = np.array(concat)
albedo = 0.726 * (concat[1,:,:] - 0.18) - 0.322 * (
concat[1,:,:] - 0.18) ** 2 - 0.015 * (concat[3,:,:] - 0.16) + 0.581 \
* (concat[3,:,:] - 0.16)
# convert albedo array to numpy array for analysis
albedo[albedo < -0.48] = None # areas outside of main image area identified set to null
with np.errstate(divide='ignore', invalid='ignore'): # ignore warning about nans in array
albedo[albedo < 0] = 0 # set any subzero pixels inside image area to 0
# collate predicted map, albedo map and projection info into xarray dataset
# 1) Retrieve projection info from uav datafile and add to netcdf
srs = osr.SpatialReference()
srs.ImportFromProj4('+init=epsg:32623')
proj_info = xr.DataArray(0, encoding={'dtype': np.dtype('int8')})
proj_info.attrs['projected_crs_name'] = srs.GetAttrValue('projcs')
proj_info.attrs['grid_mapping_name'] = 'UTM'
proj_info.attrs['scale_factor_at_central_origin'] = srs.GetProjParm('scale_factor')
proj_info.attrs['standard_parallel'] = srs.GetProjParm('latitude_of_origin')
proj_info.attrs['straight_vertical_longitude_from_pole'] = srs.GetProjParm('central_meridian')
proj_info.attrs['false_easting'] = srs.GetProjParm('false_easting')
proj_info.attrs['false_northing'] = srs.GetProjParm('false_northing')
proj_info.attrs['latitude_of_projection_origin'] = srs.GetProjParm('latitude_of_origin')
# 2) Create associated lat/lon coordinates DataArrays usig georaster (imports geo metadata without loading img)
# see georaster docs at https: // media.readthedocs.org / pdf / georaster / latest / georaster.pdf
uav = georaster.SingleBandRaster('NETCDF:"%s":Band1' % (img_file),
load_data=False)
lon, lat = uav.coordinates(latlon=True)
uav = None # close file
uav = xr.open_dataset(img_file, chunks={'x': 2000, 'y': 2000})
coords_geo = {'y': uav['y'], 'x': uav['x']}
uav = None #close file
lon_array = xr.DataArray(lon, coords=coords_geo, dims=['y', 'x'],
encoding={'_FillValue': -9999., 'dtype': 'int16', 'scale_factor': 0.000000001})
lon_array.attrs['grid_mapping'] = 'UTM'
lon_array.attrs['units'] = 'degrees'
lon_array.attrs['standard_name'] = 'longitude'
lat_array = xr.DataArray(lat, coords=coords_geo, dims=['y', 'x'],
encoding={'_FillValue': -9999., 'dtype': 'int16', 'scale_factor': 0.000000001})
lat_array.attrs['grid_mapping'] = 'UTM'
lat_array.attrs['units'] = 'degrees'
lat_array.attrs['standard_name'] = 'latitude'
# 3) add predicted map array and add metadata
predictedxr = xr.DataArray(predicted, coords=coords_geo, dims=['y','x'])
predictedxr.encoding = {'dtype': 'int16', 'zlib': True, '_FillValue': -9999}
predictedxr.name = 'Surface Class'
predictedxr.attrs['long_name'] = 'Surface classified using Random Forest'
predictedxr.attrs['units'] = 'None'
predictedxr.attrs[
'key'] = 'Snow:1; Water:2; Cryoconite:3; Clean Ice:4; Light Algae:5; Heavy Algae:6'
predictedxr.attrs['grid_mapping'] = 'UTM'
# add albedo map array and add metadata
albedoxr = xr.DataArray(albedo, coords=coords_geo, dims=['y', 'x'])
albedoxr.encoding = {'dtype': 'int16', 'scale_factor': 0.01, 'zlib': True, '_FillValue': -9999}
albedoxr.name = 'Surface albedo computed after Knap et al. (1999) narrowband-to-broadband conversion'
albedoxr.attrs['units'] = 'dimensionless'
albedoxr.attrs['grid_mapping'] = 'UTM'
# collate data arrays into a dataset
dataset = xr.Dataset({
'classified': (['x', 'y'],predictedxr),
'albedo': (['x', 'y'], albedoxr),
'Projection(UTM)': proj_info,
'longitude': (['x','y'],lon_array),
'latitude': (['x','y'],lat_array)
})
# add metadata for dataset
dataset.attrs['Conventions'] = 'CF-1.4'
dataset.attrs['Author'] = 'Joseph Cook (University of Sheffield, UK)'
dataset.attrs[
'title'] = 'Classified surface and albedo maps produced from UAV-derived multispectral ' \
'imagery of the SW Greenland Ice Sheet'
# Additional geo-referencing
dataset.attrs['nx'] = len(dataset.x)
dataset.attrs['ny'] = len(dataset.y)
dataset.attrs['xmin'] = float(dataset.x.min())
dataset.attrs['ymax'] = float(dataset.y.max())
dataset.attrs['spacing'] = 0.05
# NC conventions metadata for dimensions variables
dataset.x.attrs['units'] = 'meters'
dataset.x.attrs['standard_name'] = 'projection_x_coordinate'
dataset.x.attrs['point_spacing'] = 'even'
dataset.x.attrs['axis'] = 'x'
dataset.y.attrs['units'] = 'meters'
dataset.y.attrs['standard_name'] = 'projection_y_coordinate'
dataset.y.attrs['point_spacing'] = 'even'
dataset.y.attrs['axis'] = 'y'
# save dataset to netcdf if requested
if save_netcdf:
dataset.to_netcdf(savefig_path + "Classification_and_Albedo_Data.nc")
# plot and save figure if requested
if plot_maps or savefigs:
# set color scheme for plots - custom for predicted
cmap1 = mpl.colors.ListedColormap(
['white', 'royalblue', 'black', 'lightskyblue', 'mediumseagreen', 'darkgreen'])
cmap1.set_under(color='white') # make sure background is white
cmap2 = plt.get_cmap('Greys_r') # reverse greyscale for albedo
cmap2.set_under(color='white') # make sure background is white
fig = plt.figure(figsize=(25, 25))
plt.title("Classified ice surface and its albedos from UAV imagery: SW Greenland Ice Sheet", fontsize=28)
class_labels = ['Snow', 'Water', 'Cryoconite', 'Clean Ice', 'Light Algae', 'Heavy Algae']
# first subplot = classified map
ax1 = plt.subplot(211)
img = dataset.classified.plot(cmap=cmap1, add_colorbar=False)
cbar = fig.colorbar(mappable=img, ax=ax1)
# workaround to get colorbar labels centrally positioned
n_classes = 6
tick_locs = np.arange(1,len(class_labels),0.92)
cbar.set_ticks(tick_locs)
cbar.ax.set_yticklabels(class_labels, rotation=45, va='center')
plt.title('Classified Surface Map (UTM coordinates)'), ax1.set_aspect('equal')
# second subplot = albedo map
ax2 = plt.subplot(212)
dataset.albedo.plot(cmap=cmap2, vmin=0, vmax=1, ax=ax2), plt.title('Albedo Map (UTM coordinates)')
ax2.set_aspect('equal')
if savefigs:
plt.savefig(str(savefig_path + "UAV_classified_albedo_map.png"), dpi=150)
if plot_maps:
plt.show()
print("\n Image Classification and Albedo Function Time = ", datetime.now() - startTime)
return predicted, albedo
def __init__(self,
ds_file=None,
georaster=None,
extent=None,
lon_0=None,
projection='tmerc',
figsize=None,
fig=None,
ax=None):
"""
Create a new map.
The Map object must be initialised with georeferencing information.
This can be provided in three ways:
(1)
ds_file : str, link to a dataset understood by GDAL.
The extent of the plotting area and the lon_0 will be
set according to the properties of the dataset.
(2)
extent : (lon_lower_left,lon_upper_right,lat_lower_left,
lat_upper_right)
lon_0 : float, longitude of origin
(3)
provide a georaster.SingleBandRaster or MultiBandRaster instance
to the kwarg georaster
Creation of the matplotlib figure:
There are two options.
1) Create figure automatically.
If desired, set figsize=(x,y).
Leave fig=None and ax=None.
2) Use existing axes.
set fig=figure_obj and ax=ax_obj.
figsize is ignored.
E.g.
>>> mymap = Map(ds_file='myim.tif')
E.g.
>>> mymap = Map(extent=(-50,-48,67,68),lon_0=70)
"""
# Use handle to existing figure
if fig != None and ax != None:
self.fig = fig
self.ax = ax
# Create figure of specified size
elif figsize != None:
self.fig = plt.figure(figsize=figsize)
self.ax = plt.subplot(111)
# Create figure at system default size
else:
self.fig = plt.figure()
self.ax = plt.subplot(111)
# Get basic georeferencing info for map
# From a geoTIFF
if ds_file != None:
ds = georaster.SingleBandRaster(ds_file, load_data=False)
extent = ds.get_extent_latlon()
lon_0 = ds.srs.GetProjParm('central_meridian')
elif georaster != None:
extent = georaster.get_extent_latlon()
lon_0 = georaster.srs.GetProjParm('central_meridian')
# Otherwise check that it has been provided manually
else:
if (extent == None) or (lon_0 == None):
print('Either ds_file must be provided, or extent and lon_0.')
raise AttributeError
self.extent = extent
lonll, lonur, latll, latur = extent
# Create Basemap
self.map = Basemap(llcrnrlon=lonll,
llcrnrlat=latll,
urcrnrlon=lonur,
urcrnrlat=latur,
resolution='i',
projection=projection,
lon_0=lon_0,
lat_0=0)
print("top northing: %f" % tl_northing_TILE)
print("bottom northing: %f" % br_northing_TILE)
#dem_tile, tiles_x, tiles_y, tile_xy_dims = assign_new_tile_dims(dem_crop, tl_easting_TILE, br_easting_TILE, tl_northing_TILE, br_northing_TILE, tiles_x, tiles_y)
tiles_x, tiles_y, tile_xy_dims = assign_new_tile_dims(
dem_crop, tl_easting_TILE, br_easting_TILE, tl_northing_TILE,
br_northing_TILE, tiles_x, tiles_y)
# Write raster
#*** USE GEORASTER LIBRARY TO READ IN DTM TO THE EXTENT SPECIFIED BY THE TILE COORDINATES (tile_xy_dims) ***
#*** read me here: http://georaster.readthedocs.io/en/latest/api.html?highlight=sub#georaster.__Raster.read_single_band_subset ***
#*** no need to transform the coordinates to pixels yourself (use georaster which uses gdal to do this - probably with some bilinear interpolatin to sort the edges...) ***
#dd=georaster.SingleBandRaster(file_name, load_data=False, latlon=True)
dd_tile = georaster.SingleBandRaster(file_name,
load_data=tile_xy_dims,
latlon=False)
ofile = "%s/tile_%i.tif" % (out_path, count)
dd_tile.save_geotiff(ofile)
print("COMPLETE")
# plot tiles over main extent
def plot_it():
fig, ax = plt.subplots()
ax.plot(
(easting_min, easting_max, easting_max, easting_min, easting_min),
(northing_min, northing_min, northing_max, northing_max, northing_min),
color='red')
