def _reproject(self):
reproj_path = os.path.join(self.temp_dir, 'reproj.tif')
setattr(self, 'reprojection', reproj_path)
with rasopen(self.projection, 'r') as src:
src_profile = src.profile
src_bounds = src.bounds
src_array = src.read(1)
dst_profile = copy.deepcopy(self.target_profile)
dst_profile['dtype'] = float32
bounds = src_bounds
dst_affine, dst_width, dst_height = cdt(src_profile['crs'],
dst_profile['crs'],
src_profile['width'],
src_profile['height'], *bounds)
dst_profile.update({
'crs': dst_profile['crs'],
'transform': dst_affine,
'width': dst_width,
'height': dst_height
})
with rasopen(reproj_path, 'w', **dst_profile) as dst:
dst_array = empty((1, dst_height, dst_width), dtype=float32)
reproject(src_array,
dst_array,
src_transform=src_profile['transform'],
src_crs=src_profile['crs'],
dst_crs=self.target_profile['crs'],
dst_transform=dst_affine,
resampling=Resampling.nearest,
num_threads=2)
dst.write(
dst_array.reshape(1, dst_array.shape[1], dst_array.shape[2]))
def save(array, geometry, output_filename, crs=None, return_array=False):
try:
array = array.reshape(1, array.shape[1], array.shape[2])
except IndexError:
array = array.reshape(1, array.shape[0], array.shape[1])
geometry['dtype'] = str(array.dtype)
if crs:
geometry['crs'] = CRS({'init': crs})
with rasopen(output_filename, 'w', **geometry) as dst:
dst.write(array)
if return_array:
return array
return None
def __init__(self,
target_profile=None,
year=None,
out_dir=None,
from_file=None):
self.url_base = 'https://nassgeodata.gmu.edu/axis2/services/CDLService/' \
'GetCDLFile?year={year}&bbox={wsen}'
if from_file:
self.from_file = from_file
with rasopen(from_file) as src:
self.cdl = src.read()
self.target_profile = src.profile
self.cdl_empty = False
else:
self.cdl_empty = True
self.cdl = None
if not out_dir:
self.cdl_location = os.path.join(os.path.dirname(__file__),
'model_data')
else:
self.cdl_location = out_dir
self.zip_file = os.path.join(self.cdl_location,
'{}_30m_cdls.zip'.format(year))
self.temp_dir = mkdtemp()
if target_profile and year:
self.target_profile = target_profile
self.bbox = RasterBounds(
profile=self.target_profile,
affine_transform=self.target_profile['transform'])
self.bbox.expand(**{
'east': 0.1,
'west': -0.1,
'north': 0.2,
'south': -0.2
})
self.bbox_projected = bb = self.bbox.to_lambert_conformal_conic(
)
bb_str = '{},{},{},{}'.format(bb[0], bb[1], bb[2], bb[3])
self.request_url = self.url_base.format(year=year, wsen=bb_str)
self.data_url = self._get_data_url()
self.original_tif = None
self.mask = None
self.projection = None
self.reprojection = None
def _resample(self):
resample_path = os.path.join(self.temp_dir, 'resample.tif')
with rasopen(self.mask, 'r') as src:
array = src.read(1)
profile = src.profile
res = src.res
try:
target_affine = self.target_profile['affine']
except KeyError:
target_affine = self.target_profile['transform']
target_res = target_affine.a
res_coeff = res[0] / target_res
new_array = empty(shape=(1, round(array.shape[0] * res_coeff),
round(array.shape[1] * res_coeff)), dtype=float32)
aff = src.transform
new_affine = Affine(aff.a / res_coeff, aff.b, aff.c, aff.d, aff.e / res_coeff, aff.f)
profile['transform'] = self.target_profile['transform']
profile['width'] = self.target_profile['width']
profile['height'] = self.target_profile['height']
profile['dtype'] = str(new_array.dtype)
delattr(self, 'mask')
with rasopen(resample_path, 'w', **profile) as dst:
reproject(array, new_array, src_transform=aff, dst_transform=new_affine, src_crs=src.crs,
dst_crs=src.crs, resampling=Resampling.nearest)
dst.write(new_array)
with rasopen(resample_path, 'r') as src:
arr = src.read()
return arr
def setUp(self):
self.box = (-109.9849, 46.46738, -109.93647, 46.498625)
self.dst_srs = '26912'
self.kwargs = dict([('dst_crs', self.dst_srs)])
self.kwargs_centroid = dict([('dst_crs', self.dst_srs),
('centroid',
((self.box[1] + self.box[3]) / 2,
((self.box[0] + self.box[2]) / 2))),
('buffer', 1700)])
self.tile_loc = os.path.join(os.path.dirname(__file__), 'data',
'wheatland_tile.tif')
with rasopen(self.tile_loc, 'r') as src:
self.profile = src.profile
self.array = src.read()
def save(self, array, geometry, output_filename, crs=None):
array = array.reshape(geometry['count'], array.shape[1],
array.shape[2])
geometry['dtype'] = uint8
if crs:
dst_crs = CRS({'init': 'epsg:{}'.format(crs)})
if geometry['crs'] != dst_crs:
self.reproject_multiband(output_filename, dst_crs)
return None
with rasopen(output_filename, 'w', **geometry) as dst:
dst.write(array)
return None
def mask_raster_to_features(raster, features, features_meta):
# This function is useful when you don't have access to the
# file from which the features came or if the file doesn't exist.
gdf = gpd.GeoDataFrame.from_features(features, features_meta) # do I need
# the whole metadata?
gdf = gdf[gdf.geometry.notnull()]
with rasopen(raster, 'r') as src:
crs = CRS(src.crs['init'])
print(crs)
shp = gdf.to_crs(src.crs)
features = get_features(shp)
arr = src.read()
out_image, out_transform = mask(src, shapes=features)
out_image[out_image != 0] = 1
meta = src.meta
return out_image, meta
def test_conforming_array(self):
""" Test shape of Gridmet vs. Landsat image.
:return:
"""
l8 = Landsat8(self.dir_name_LC8)
shape = 1, l8.rasterio_geometry['height'], l8.rasterio_geometry[
'width']
polygon = l8.get_tile_geometry()
cdl = Cdl(year=self.year,
target_profile=l8.profile,
out_dir=self.dir_name_LC8)
_ = cdl.get_conforming_data(polygon)
with rasopen(os.path.join(self.dir_name_LC8, 'cdl.tif')) as dst:
arr = dst.read()
self.assertEqual(arr.shape, shape)
def _point_raster_extract(self, raster, _name):
with rasopen(raster, 'r') as rsrc:
rass_arr = rsrc.read()
rass_arr = rass_arr.reshape(rass_arr.shape[1], rass_arr.shape[2])
affine = rsrc.transform
s = Series(index=range(0, self.extracted_points.shape[0]), name=_name)
for ind, row in self.extracted_points.iterrows():
x, y = self._geo_point_to_projected_coords(row['X'], row['Y'])
c, r = ~affine * (x, y)
try:
raster_val = rass_arr[int(r), int(c)]
s[ind] = float(raster_val)
except IndexError:
s[ind] = None
return s
def clip_raster(evaluated, path, row, outfile=None):
out = _get_path_row_geometry(path, row)
with rasopen(evaluated, 'r') as src:
out = out.to_crs(src.crs['init'])
features = get_features(out)
# if crop == true for mask, you have to update the metadata.
out_image, out_transform = mask(src, shapes=features, crop=True, nodata=np.nan)
meta = src.meta.copy()
count = out_image.shape[0]
meta.update({"driver": "GTiff",
"height": out_image.shape[1],
"width": out_image.shape[2],
"transform": out_transform})
if outfile is not None:
save_raster(out_image, outfile, meta, count)
def test_warped_vrt(self):
warped_vrt.warp_vrt(self.directory)
shapes = []
dirs = [
_ for _ in os.listdir(self.directory) if not _.endswith('.txt')
]
for d in dirs:
lst = [
_ for _ in os.listdir(os.path.join(self.directory, d))
if _.endswith('.TIF')
]
for l in lst:
tif = os.path.join(self.directory, d, l)
with rasopen(tif, 'r') as src:
shapes.append(src.shape)
shutil.rmtree(self.directory)
self.assertEqual(shapes[0], shapes[1])
def write_raster(self, out_file, new_array=None):
if isinstance(new_array, ndarray):
self.new_array = new_array
try:
self.new_array = self.new_array.reshape(1, self.new_array.shape[1],
self.new_array.shape[2])
except IndexError:
self.new_array = self.new_array.reshape(1, self.new_array.shape[0],
self.new_array.shape[1])
self.raster_geo['dtype'] = str(self.new_array.dtype)
self.raster_geo['count'] = 1
with rasopen(out_file, 'w', **self.raster_geo) as dst:
dst.write(self.new_array)
return None
def get_image(self, state):
""" Get NAIP imagery from states excluding Hawaii and Alaska
Current hack in this method and in GeoBounds is hard-coded epsg: 3857 'web mercator',
though the NAIP service provides epsg: 102100 a deprecated ESRI SRS'
:param state: e.g. 'ND'
:param size: tuple of horizontal by vertical size in pixels, e.g., (512, 512)
:return:
"""
coords = {
x: y
for x, y in zip(['west', 'south', 'east', 'north'], self.bbox)
}
w, s, e, n = GeoBounds(**coords).to_web_mercator()
self.web_mercator_bounds = (w, s, e, n)
bbox_str = self.bounds_fmt.format(w=w, s=s, e=e, n=n)
naip_str = '{}_NAIP'.format(state)
query = self.usda_query_str.format(naip_str, bbox_str)
url = '{}{}'.format(self.naip_base_url, query)
req = get(url, verify=False, stream=True)
if req.status_code != 200:
raise ValueError('Bad response {} from NAIP API request.'.format(
req.status_code))
# with open(self.temp_file, 'wb') as f:
# f.write(req.content)
with open(self.temp_file, 'wb') as f:
for chunk in req.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
with rasopen(self.temp_file, 'r') as src:
array = src.read()
profile = src.profile
return array, profile
def mask_raster_to_shapefile(shapefile, raster, return_binary=True):
'''
Generates a mask with 1 everywhere
shapefile data is present and a no_data value everywhere else.
no_data is -1 in this case, as it is never a valid class label.
Switching coordinate reference systems is important here, or
else the masking won't work.
'''
shp = gpd.read_file(shapefile)
shp = shp[shp.geometry.notnull()]
with rasopen(raster, 'r') as src:
# pyproj deprecated the +init syntax.
crs = CRS(src.crs['init'])
shp = shp.to_crs(crs)
features = get_features(shp)
arr = src.read()
out_image, out_transform = mask(src, shapes=features, filled=False)
if return_binary:
out_image[out_image != 0] = 1
meta = src.meta
return out_image, meta
def __init__(self, raster=None, affine_transform=None, profile=None, latlon=True):
BBox.__init__(self)
if raster:
with rasopen(raster, 'r') as src:
profile = src.profile
affine = profile['transform']
if affine_transform:
affine = affine_transform
col, row = 0, 0
w, n = affine * (col, row)
col, row = profile['width'], profile['height']
e, s = affine * (col, row)
if latlon and profile['crs'] != CRS({'init': 'epsg:4326'}):
in_proj = Proj(init=profile['crs']['init'])
self.west, self.north = in_proj(w, n, inverse=True)
self.east, self.south = in_proj(e, s, inverse=True)
else:
self.north, self.west, self.south, self.east = n, w, s, e
def get_mask(self, clip_geometry=None, out_file=None):
arr = None
if self.cdl_empty:
try:
arr = self.get_conforming_data(clip_geometry=clip_geometry)
except ValueError:
print('Need clip geometry to build cdl')
else:
arr = self.cdl
crop = list(self.crop.keys())
msk = isin(arr, crop)
msk = ~msk
msk = msk.astype(uint8)
profile = copy.deepcopy(self.target_profile)
profile['dtype'] = uint8
if out_file:
with rasopen(out_file, 'w', **profile) as dst:
dst.write(msk)
return msk
def iterate_over_image_and_evaluate_patchwise_lstm_cnn(image_stack,
model_path,
out_filename,
out_meta,
n_classes,
tile_size=24):
model = load_model(model_path, custom_objects={'m_acc': m_acc})
timeseries = []
for i in range(0, image_stack.shape[0] - 3, 3):
timeseries.append(image_stack[i:i + 3])
timeseries = np.asarray(timeseries)
timeseries = np.swapaxes(timeseries, 1, 3)
timeseries = np.expand_dims(timeseries, 0)
print(timeseries.shape)
for start_idx in range(0, timeseries.shape[1] - 12):
predictions = np.zeros(
(timeseries.shape[2], timeseries.shape[3], n_classes))
timeseries_copy = timeseries[:, start_idx:start_idx + 12, :, :, :]
for i in range(0, timeseries_copy.shape[2] - tile_size, tile_size):
for j in range(0, timeseries_copy.shape[3] - tile_size, tile_size):
image_tile = timeseries_copy[:, :, i:i + tile_size,
j:j + tile_size, :]
if np.all(image_tile == 0):
continue
preds = np.squeeze(model.predict(image_tile))
predictions[i:i + tile_size,
j:j + tile_size, :] = np.sum(preds, axis=0)
stdout.write("{}, {:.3f}\r".format(start_idx,
i / timeseries_copy.shape[2]))
predictions = np.swapaxes(predictions, 0, 2)
out_meta.update({'count': n_classes, 'dtype': np.float64})
with rasopen(out_filename, "w", **out_meta) as dst:
dst.write(predictions)
def reproject_tiles(self):
reproj_path = os.path.join(self.temp_dir, 'tiled_reproj.tif')
setattr(self, 'reprojection', reproj_path)
profile = copy.deepcopy(self.target_profile)
profile['dtype'] = float32
bb = self.web_mercator_bounds
bounds = (bb[0], bb[1], bb[2], bb[3])
dst_affine, dst_width, dst_height = calculate_default_transform(
self.merged_profile['crs'], profile['crs'],
self.merged_profile['width'], self.merged_profile['height'],
*bounds)
profile.update({
'crs': profile['crs'],
'transform': dst_affine,
'width': dst_width,
'height': dst_height
})
with rasopen(reproj_path, 'w', **profile) as dst:
dst_array = empty((1, dst_height, dst_width), dtype=float32)
reproject(self.merged_array,
dst_array,
src_transform=self.merged_transform,
src_crs=self.merged_profile['crs'],
dst_crs=self.target_profile['crs'],
dst_transform=dst_affine,
resampling=Resampling.cubic,
num_threads=2)
dst.write(
dst_array.reshape(1, dst_array.shape[1], dst_array.shape[2]))
delattr(self, 'merged_array')
def stack_images_from_list_of_filenames_sorted_by_date(filenames):
filenames = sorted(filenames, key=lambda x: parse_date(x))
dates = [parse_date(x) for x in filenames]
# if len(filenames) > 16:
# filenames = filenames[:16]
first = True
image_stack = None
i = 0
n_bands = 7
if not len(filenames):
print('empty list of filenames')
return (None, None, None, None)
for filename in filenames:
with rasopen(filename, 'r') as src:
arr = src.read()
meta = deepcopy(src.meta)
if first:
first = False
image_stack = np.zeros((n_bands * len(filenames) + len(filenames),
arr.shape[1], arr.shape[2]),
dtype=np.int16)
target_meta = deepcopy(meta)
target_fname = filename
image_stack[0:n_bands] = arr
i += n_bands
else:
try:
image_stack[i:i + n_bands] = arr
i += n_bands
except ValueError as e:
arr = warp_single_image(filename, target_meta)
image_stack[i:i + n_bands] = arr
i += n_bands
image_stack[-len(filenames):] = date_stack(dates, image_stack.shape)
return image_stack, target_meta, target_fname, meta
def save_raster(arr, outfile, meta, count=5):
meta.update(count=count)
with rasopen(outfile, 'w', **meta) as dst:
dst.write(arr)
def _get_mask_from_raster(self, extra_mask):
with rasopen(extra_mask, mode='r') as src:
arr = src.read()
self.raster_geo = src.meta.copy()
return arr
def load_raster(raster_name):
with rasopen(raster_name, 'r') as src:
arr = src.read()
meta = src.meta.copy()
return arr, meta
def _load_image(f, image=False):
with rasopen(f, 'r') as src:
im = src.read()
return im
def _get_crs(self):
for key, val in self.paths_map.items():
with rasopen(val, 'r') as src:
crs = src.crs
break
return crs
from rasterio import open as rasopen
import matplotlib.pyplot as plt
import numpy as np
import sys
raster = sys.argv[1]
with rasopen(raster) as src:
arr = np.squeeze(src.read())
src = None
print(arr.shape)
print(arr.dtype)
arr[arr == -9999] = np.nan
ts = 5000
for i in range(0, arr.shape[0] - ts, ts):
for j in range(0, arr.shape[1] - ts, ts):
plt.imshow(arr[i:i + ts, j:j + ts])
plt.colorbar()
plt.show()
def warp_vrt(directory,
delete_extra=False,
use_band_map=False,
overwrite=False,
remove_bqa=True):
""" Read in image geometry, resample subsequent images to same grid.
The purpose of this function is to snap many Landsat images to one geometry. Use Landsat578
to download and unzip them, then run them through this to get identical geometries for analysis.
Files
:param use_band_map:
:param delete_extra:
:param directory: A directory containing sub-directories of Landsat images.
:return: None
"""
if 'resample_meta.txt' in os.listdir(directory) and not overwrite:
print('{} has already had component images warped'.format(directory))
return None
mapping = {'LC8': Landsat8, 'LE7': Landsat7, 'LT5': Landsat5}
vrt_options = {}
list_dir = [
x[0] for x in os.walk(directory)
if os.path.basename(x[0])[:3] in mapping.keys()
]
extras = [
os.path.join(directory, x) for x in os.listdir(directory)
if x.endswith('.tif')
]
first = True
for d in list_dir:
sat = LandsatImage(d).satellite
paths = extras
root = os.path.join(directory, d)
if os.path.isdir(root):
for x in os.listdir(root):
if remove_bqa and x.endswith('BQA.TIF'):
try:
os.remove(x)
except FileNotFoundError:
pass
elif use_band_map:
bands = BandMap().selected
for y in bands[sat]:
if x.endswith('B{}.TIF'.format(y)):
paths.append(os.path.join(directory, d, x))
else:
if x.endswith('.TIF') or x.endswith('.tif'):
paths.append(os.path.join(directory, d, x))
if x.endswith('MTL.txt'):
mtl = os.path.join(directory, d, x)
if first:
landsat = mapping[sat](os.path.join(directory, d))
dst = landsat.rasterio_geometry
vrt_options = {
'resampling': Resampling.nearest,
'dst_crs': dst['crs'],
'dst_transform': dst['transform'],
'dst_height': dst['height'],
'dst_width': dst['width']
}
message = """
This directory has been resampled to same grid.
Master grid is {}.
{}
""".format(d, datetime.now())
with open(os.path.join(directory, 'resample_meta.txt'), 'w') as f:
f.write(message)
first = False
os.rename(mtl, mtl.replace('.txt', 'copy.txt'))
for tif_path in paths:
print('warping {}'.format(os.path.basename(tif_path)))
with rasopen(tif_path, 'r') as src:
with WarpedVRT(src, **vrt_options) as vrt:
data = vrt.read()
dst_dir, name = os.path.split(tif_path)
outfile = os.path.join(dst_dir, name)
meta = vrt.meta.copy()
meta['driver'] = 'GTiff'
with rasopen(outfile, 'w', **meta) as dst:
dst.write(data)
os.rename(mtl.replace('.txt', 'copy.txt'), mtl)
if delete_extra:
for x in os.listdir(os.path.join(directory, d)):
x_file = os.path.join(directory, d, x)
if x_file not in paths:
if x[-7:] not in ['ask.tif', 'MTL.txt']:
print('removing {}'.format(x_file))
os.remove(x_file)
def point_target_extract(points, nlcd_path,
target_shapefile=None, count_limit=None):
point_data = {}
with fopen(points, 'r') as src:
for feature in src:
name = feature['id']
proj_coords = feature['geometry']['coordinates']
point_data[name] = {'point': feature['geometry'],
'coords': proj_coords}
# point_crs = src.profile['crs']['init']
pt_ct = 0
for pt_id, val in point_data.items():
pt_ct += 1
if pt_ct < count_limit:
pt = shape(val['point'])
with fopen(target_shapefile, 'r') as target_src:
has_attr = False
for t_feature in target_src:
polygon = t_feature['geometry']
if pt.within(shape(polygon)):
print('pt id {}, props: {}'
.format(pt_id, t_feature['properties']))
props = t_feature['properties']
point_data[pt_id]['properties'] = {'IType': props['IType'],
'LType': props['LType']}
has_attr = True
break
if not has_attr:
if nlcd_path:
with rasopen(nlcd_path, 'r') as rsrc:
rass_arr = rsrc.read()
rass_arr = rass_arr.reshape(rass_arr.shape[1], rass_arr.shape[2])
affine = rsrc.affine
x, y = val['coords']
col, row = ~affine * (x, y)
raster_val = rass_arr[int(row), int(col)]
ltype_dct = {'IType': None,
'LType': str(raster_val)}
point_data[pt_id]['properties'] = ltype_dct
print('id {} has no FLU, '
'nlcd {}'.format(pt_id,
nlcd_value(ltype_dct['LType'])))
else:
ltype_dct = {'IType': None,
'LType': None}
point_data[pt_id]['properties'] = ltype_dct
idd = []
ltype = []
itype = []
x = []
y = []
ct = 0
for pt_id, val in point_data.items():
ct += 1
if ct < count_limit:
idd.append(pt_id)
ltype.append(val['properties']['LType'])
itype.append(val['properties']['IType'])
x.append(val['coords'][0])
y.append(val['coords'][1])
else:
break
dct = dict(zip(['ID', 'LTYPE', 'ITYPE', 'X', 'Y'],
[idd, ltype, itype, x, y]))
df = DataFrame(data=dct)
return df
def __init__(self, obj):
'''
:param obj: Directory containing an unzipped Landsat 5, 7, or 8 image. This should include at least
a tif for each band, and a .mtl file.
'''
self.obj = obj
if os.path.isdir(obj):
self.isdir = True
self.date_acquired = None
self.file_list = os.listdir(obj)
self.tif_list = [x for x in os.listdir(obj) if x.endswith('.TIF')]
self.tif_list.sort()
# parse metadata file into attributes
# structure: {HEADER: {SUBHEADER: {key(attribute), val(attribute value)}}}
self.mtl = mtl.parsemeta(obj)
self.meta_header = list(self.mtl)[0]
self.super_dict = self.mtl[self.meta_header]
for key, val in self.super_dict.items():
for sub_key, sub_val in val.items():
# print(sub_key.lower(), sub_val)
setattr(self, sub_key.lower(), sub_val)
self.satellite = self.landsat_scene_id[:3]
# create numpy nd_array objects for each band
self.band_list = []
self.tif_dict = {}
for i, tif in enumerate(self.tif_list):
raster = os.path.join(self.obj, tif)
# set all lower case attributes
tif = tif.lower()
front_ind = tif.index('b')
end_ind = tif.index('.tif')
att_string = tif[front_ind:end_ind]
self.band_list.append(att_string)
self.tif_dict[att_string] = raster
self.band_count = i + 1
if i == 0:
with rasopen(raster) as src:
transform = src.transform
profile = src.profile
meta = src.meta.copy()
self.rasterio_geometry = meta
self.profile = profile
self.transform = transform
self.shape = (1, profile['height'], profile['width'])
bounds = RasterBounds(affine_transform=transform,
profile=profile,
latlon=False)
self.bounds = bounds
self.north, self.west, self.south, self.east = bounds.get_nwse_tuple(
)
self.coords = bounds.as_tuple('nsew')
self.solar_zenith = 90. - self.sun_elevation
self.solar_zenith_rad = self.solar_zenith * pi / 180
self.sun_elevation_rad = self.sun_elevation * pi / 180
self.earth_sun_dist = self.earth_sun_d(self.date_acquired)
dtime = datetime.strptime(str(self.date_acquired), '%Y-%m-%d')
julian_day = dtime.strftime('%j')
self.doy = int(julian_day)
self.scene_coords_deg = self._scene_centroid()
self.scene_coords_rad = deg2rad(self.scene_coords_deg[0]), deg2rad(
self.scene_coords_deg[1])
