def _get_stack_channels(self):
stack = None
first = True
for i, feat in enumerate(self.data.paths_map.keys()):
self.feature_ras = self.data.paths_map[feat]
with rasopen(self.feature_ras, mode='r') as src:
arr = src.read()
self.raster_geo = src.meta.copy()
if self.saved_array:
break
if first:
first_geo = deepcopy(self.raster_geo)
empty = zeros((len(self.data.paths_map.keys()), arr.shape[1], arr.shape[2]), float16)
stack = empty
stack[i, :, :] = self.normalize_image_channel(arr)
first = False
else:
try:
stack[i, :, :] = self.normalize_image_channel(arr)
except ValueError:
arr = warp_single_image(self.feature_ras, first_geo)
stack[i, :, :] = self.normalize_image_channel(arr)
return stack
def concatenate_fmasks(image_directory,
class_mask,
class_mask_geo,
nodata=0,
target_directory=None):
''' ``Fmasks'' are masks of clouds and water. We don't want clouds/water in
the training set, so this function gets all the fmasks for a landsat
scene (contained in image_directory), and merges them into one raster.
They may not be the same size, so warp_vrt is used to make them align.
'''
class_mask = class_mask.copy()
paths = []
for dirpath, dirnames, filenames in os.walk(image_directory):
for f in filenames:
for suffix in mask_rasters():
if f.endswith(suffix):
pth = os.path.join(dirpath, f)
paths.append(pth)
for fmask_file in paths:
fmask, _ = load_raster(fmask_file)
# clouds, water present where fmask == 1.
try:
class_mask = ma.masked_where(fmask == 1, class_mask)
except (ValueError, IndexError) as e:
fmask = warp_single_image(fmask_file, class_mask_geo)
class_mask = ma.masked_where(fmask == 1, class_mask)
return class_mask
def concatenate_fmasks_single_scene(class_labels, image_directory, target_date,
class_mask_geo):
date = None
for d in os.listdir(image_directory):
if os.path.isdir(os.path.join(image_directory, d)):
try:
date = _parse_landsat_capture_date(d)
except Exception as e:
print(e)
continue
if date == target_date:
landsat_directory = d
break
class_mask = class_labels.copy()
paths = []
for dirpath, dirnames, filenames in os.walk(landsat_directory):
for f in filenames:
for suffix in mask_rasters():
if f.endswith(suffix):
pth = os.path.join(dirpath, f)
paths.append(pth)
for fmask_file in paths:
fmask, _ = load_raster(fmask_file)
# clouds, water present where fmask == 1.
try:
class_mask = ma.masked_where(fmask == 1, class_mask)
except (ValueError, IndexError) as e:
fmask = warp_single_image(fmask_file, class_mask_geo)
class_mask = ma.masked_where(fmask == 1, class_mask)
return class_mask
def stack_rasters(paths_map, target_geo, target_shape):
first = True
stack = None
num_rasters = 0
for key in paths_map: num_rasters += len(paths_map[key])
j = 0
for feat in sorted(paths_map.keys()): # ensures the stack is in the same order each time.
# Ordering within bands is assured by sorting the list that
# each band corresponding to, as that's sorting by date.
feature_rasters = paths_map[feat] # maps bands to their location in filesystem.
for feature_raster in feature_rasters:
with rasopen(feature_raster, mode='r') as src:
arr = src.read()
if first:
stack = np.zeros((num_rasters, target_shape[1], target_shape[2]), np.uint16)
stack[j, :, :] = arr
j += 1
first = False
else:
try:
stack[j, :, :] = arr
j += 1
except ValueError:
arr = warp_single_image(feature_raster, target_geo)
stack[j, :, :] = arr
j += 1
return stack
def create_image_stack(paths_map):
first = True
stack = None
j = 0
num_rasters = 0
for ls in paths_map.values():
num_rasters += len(ls)
print(num_rasters)
for feat in sorted(paths_map.keys()):
feature_rasters = paths_map[feat]
for feature_raster in feature_rasters:
if first:
arr, target_geo = load_raster(feature_raster)
stack = np.zeros((num_rasters, arr.shape[1], arr.shape[2]), np.uint16)
stack[j, :, :] = arr
j += 1
first = False
else:
try:
arr = load_raster(feature_raster)
stack[j, :, :] = arr
j += 1
except ValueError:
arr = warp_single_image(feature_raster, target_geo)
stack[j, :, :] = arr
j += 1
return stack
def _maybe_warp(feature_raster, target_geo, target_shape):
arr, _ = load_raster(feature_raster)
if arr.shape != target_shape:
print("#####################")
print(arr.shape)
print(feature_raster, target_geo)
arr = warp_single_image(feature_raster, target_geo)
print(arr.shape)
return arr, feature_raster
def extract_training_data_over_path_row(test_train_shapefiles,
path,
row,
year,
image_directory,
training_data_root_directory,
n_classes,
assign_shapefile_class_code,
preprocessing_func=None,
tile_size=608,
use_fmasks=False,
use_cdl=False):
if not isinstance(test_train_shapefiles, dict):
raise ValueError("expected dict, got {}".format(
type(test_train_shapefiles)))
path_row_year = str(path) + '_' + str(row) + '_' + str(year)
image_path = os.path.join(image_directory, path_row_year)
if not os.path.isdir(image_path):
download_from_pr(path, row, year, image_directory)
image_path_maps = paths_map_multiple_scenes(image_path)
mask_file = _random_tif_from_directory(image_path)
mask, mask_meta = load_raster(mask_file)
mask = np.zeros_like(mask).astype(np.int)
if use_cdl:
cdl_path = os.path.join(image_path, 'cdl_mask.tif')
cdl_raster, cdl_meta = load_raster(cdl_path)
if mask.shape != cdl_raster.shape:
cdl_raster = warp_single_image(cdl_path, mask_meta)
cdl_raster = np.swapaxes(cdl_raster, 0, 2)
try:
image_stack = create_image_stack(image_path_maps)
except CRSError as e:
print(e)
return
image_stack = np.swapaxes(image_stack, 0, 2)
for key, shapefiles in test_train_shapefiles.items():
if key.lower() not in ('test', 'train'):
raise ValueError(
"expected key to be one of case-insenstive {test, train},\
got {}".format(key))
training_data_directory = os.path.join(training_data_root_directory,
key)
class_labels = create_class_labels(shapefiles,
assign_shapefile_class_code)
if use_fmasks:
class_labels = concatenate_fmasks(image_path, class_labels,
mask_meta)
class_labels = np.swapaxes(class_labels, 0, 2)
class_labels = np.squeeze(class_labels)
tiles_y, tiles_x = _target_indices_from_class_labels(
class_labels, tile_size)
_save_training_data_from_indices(image_stack, class_labels,
training_data_directory, n_classes,
tiles_x, tiles_y, tile_size)
def stack_images_from_list_of_filenames_sorted_by_date(filenames):
filenames = sorted(filenames, key=lambda x: parse_date(x))
dates = [parse_date(x) for x in filenames]
# if len(filenames) > 16:
# filenames = filenames[:16]
first = True
image_stack = None
i = 0
n_bands = 7
if not len(filenames):
print('empty list of filenames')
return (None, None, None, None)
for filename in filenames:
with rasopen(filename, 'r') as src:
arr = src.read()
meta = deepcopy(src.meta)
if first:
first = False
image_stack = np.zeros((n_bands * len(filenames) + len(filenames),
arr.shape[1], arr.shape[2]),
dtype=np.int16)
target_meta = deepcopy(meta)
target_fname = filename
image_stack[0:n_bands] = arr
i += n_bands
else:
try:
image_stack[i:i + n_bands] = arr
i += n_bands
except ValueError as e:
arr = warp_single_image(filename, target_meta)
image_stack[i:i + n_bands] = arr
i += n_bands
image_stack[-len(filenames):] = date_stack(dates, image_stack.shape)
return image_stack, target_meta, target_fname, meta
def check_shape(self, var, path):
if not var.shape == self.shape:
new = warped_vrt.warp_single_image(image_path=path, profile=self.profile, resampling='nearest')
return new
else:
return var
def extract_training_data_over_path_row_single_scene(
test_train_shapefiles,
path,
row,
year,
image_directory,
training_data_root_directory,
n_classes,
assign_shapefile_class_code,
preprocessing_func=None,
tile_size=608):
if not isinstance(test_train_shapefiles, dict):
raise ValueError("expected dict, got {}".format(
type(test_train_shapefiles)))
path_row_year = str(path) + '_' + str(row) + '_' + str(year)
image_path = os.path.join(image_directory, path_row_year)
if not os.path.isdir(image_path):
download_from_pr(path, row, year, image_directory)
image_path_maps = paths_mapping_single_scene(image_path)
mask_file = _random_tif_from_directory(image_path)
mask, mask_meta = load_raster(mask_file)
mask = np.zeros_like(mask).astype(np.int)
cdl_path = os.path.join(image_path, 'cdl_mask.tif')
cdl_raster, cdl_meta = load_raster(cdl_path)
if mask.shape != cdl_raster.shape:
cdl_raster = warp_single_image(cdl_path, mask_meta)
cdl_raster = np.swapaxes(cdl_raster, 0, 2)
for key, shapefiles in test_train_shapefiles.items():
try:
class_labels = create_class_labels(shapefiles,
assign_shapefile_class_code,
mask_file)
except TypeError as e:
print(image_directory)
download_from_pr(path, row, year, image_directory)
print(e)
if key.lower() not in ('test', 'train'):
raise ValueError(
"expected key to be one of case-insenstive {test, train},\
got {}".format(key))
begin = datetime.date(year=year, month=6, day=15)
end = datetime.date(year=year, month=9, day=1)
for date, paths_map in image_path_maps.items():
if date < begin or date > end:
print('skipping:', date)
continue
try:
date_raster = _days_from_january_raster(
date, target_shape=mask.shape)
date_raster = np.swapaxes(date_raster, 0, 2)
image_stack = stack_rasters_single_scene(
paths_map, target_geo=mask_meta, target_shape=mask.shape)
image_stack = np.swapaxes(image_stack, 0, 2)
image_stack = np.dstack((image_stack, date_raster))
except RasterioIOError as e:
print("Redownload images for", path_row_year)
print(e)
return
training_data_directory = os.path.join(
training_data_root_directory, key)
class_labels_single_scene = concatenate_fmasks_single_scene(
class_labels, image_path, date, mask_meta)
class_labels_single_scene = np.swapaxes(class_labels_single_scene,
0, 2)
class_labels_single_scene = np.squeeze(class_labels_single_scene)
tiles_y, tiles_x = _target_indices_from_class_labels(
class_labels_single_scene, tile_size)
_save_training_data_from_indices(image_stack,
class_labels_single_scene,
cdl_raster,
training_data_directory,
n_classes, tiles_x, tiles_y,
tile_size)
def _maybe_warp(feature_raster, target_geo, target_shape):
arr, _ = load_raster(feature_raster)
if not arr.shape == target_shape:
arr = warp_single_image(feature_raster, target_geo)
return arr, feature_raster
def extract_training_data_over_path_row(test_train_shapefiles,
path,
row,
year,
image_directory,
training_data_root_directory,
n_classes,
assign_shapefile_class_code,
path_map_func=None,
preprocessing_func=None,
tile_size=608):
if path_map_func is None:
path_map_func = paths_map_multiple_scenes
if not isinstance(test_train_shapefiles, dict):
raise ValueError("expected dict, got {}".format(
type(test_train_shapefiles)))
path_row_year = str(path) + '_' + str(row) + '_' + str(year)
image_path = os.path.join(image_directory, path_row_year)
if not os.path.isdir(image_path):
download_from_pr(path, row, year, image_directory)
image_path_maps = path_map_func(image_path)
mask_file = _random_tif_from_directory(image_path)
mask, mask_meta = load_raster(mask_file)
mask = np.zeros_like(mask).astype(np.int)
cdl_path = os.path.join(image_path, 'cdl_mask.tif')
cdl_raster, cdl_meta = load_raster(cdl_path)
if mask.shape != cdl_raster.shape:
cdl_raster = warp_single_image(cdl_path, mask_meta)
cdl_raster = np.swapaxes(cdl_raster, 0, 2)
try:
image_stack = stack_rasters_multiprocess(image_path_maps,
target_geo=mask_meta,
target_shape=mask.shape)
image_stack = np.swapaxes(image_stack, 0, 2)
except RasterioIOError as e:
print("Redownload images for", path_row_year)
print(e)
return
for key, shapefiles in test_train_shapefiles.items():
if key.lower() not in ('test', 'train'):
raise ValueError(
"expected key to be one of case-insenstive {test, train},\
got {}".format(key))
training_data_directory = os.path.join(training_data_root_directory,
key)
first = True
class_labels = None
for f in shapefiles:
class_code = assign_shapefile_class_code(f)
print(f, class_code)
out, _ = mask_raster_to_shapefile(f,
mask_file,
return_binary=False)
if first:
class_labels = out
class_labels[~class_labels.mask] = class_code
first = False
else:
class_labels[~out.mask] = class_code
class_labels = concatenate_fmasks(image_path, class_labels, mask_meta)
class_labels = np.swapaxes(class_labels, 0, 2)
class_labels = np.squeeze(class_labels)
tiles_y, tiles_x = _target_indices_from_class_labels(
class_labels, tile_size)
_save_training_data_from_indices(image_stack, class_labels, cdl_raster,
training_data_directory, n_classes,
tiles_x, tiles_y, tile_size)