def cloud_mask(bands_directory, cloud_threshold=1500,
shadow_threshold=20, save=False):
"""
Get cloud mask from bands of the image in <directory>
according to thresholding images by red, green and blue channels
:param directory: The directory of images
:param threhold: If red, green and blue values of the pixel
are greater than <threshold> to be considered
as cloud pixel
:return: 2D array with 1s and 0s, where 0s corespond to cloudy pixels
1s - to cloud-free pixels.
"""
red_band_name = band_name(bands_directory, Bands.RED)
green_band_name = band_name(bands_directory, Bands.GREEN)
blue_band_name = band_name(bands_directory, Bands.BLUE)
red_band = read_array(red_band_name)
green_band = read_array(green_band_name)
blue_band = read_array(blue_band_name)
cloud_mask = ((red_band > cloud_threshold) * (green_band > cloud_threshold)
* (blue_band > cloud_threshold))
shadow_mask = ((red_band < shadow_threshold)
* (green_band < shadow_threshold)
* (blue_band < shadow_threshold))
mask = ~(cloud_mask + shadow_mask)
mask = mask.astype('int')
if save:
with open('{}/cloud_mask.pkl'.format(bands_directory), 'wb') as f:
pkl.dump(mask, f)
return mask
def main(args):
print("Search colored image...")
colored_image_name = band_name(args.product, Bands.TCI, extension='.jp2')
if colored_image_name is not None:
print("There is colored image")
image = read_array(colored_image_name).transpose(1, 2, 0)
else:
print("There isn't colored image. Try mix...\n")
image = save_color_image(args.product, Bands.RED,
Bands.GREEN, Bands.BLUE,
bright_limit=args.bright_limit,
output_file=args.output)
if args.plot or args.save is not None:
print("\nPrepare plot...")
plt.figure(figsize=args.size)
plt.imshow(image)
if args.plot:
print("\nPlot...")
plt.show()
if args.save is not None:
print("\nSave in {}...".format(args.save))
plt.savefig(args.save)
return image
def NDVI(path, size=None):
"""
Calculates NDVI for product
:param path: str, path to product.
:param size: (height, width), output size.
If not given, use size of bands.
:return: array, NDVI
"""
NIR = read_array(band_name(path, Bands.NIR))
RED = read_array(band_name(path, Bands.RED))
if size is None:
size = max(NIR.shape, RED.shape)
NIR = resize_band(NIR, size)
RED = resize_band(RED, size)
return (NIR.astype('float') - RED.astype('float')) / (
NIR.astype('float') + RED.astype('float'))
def debug_forest_probability(path):
"""
Shows image, cloud mask, probabilities and NDVI distribution.
:param path: str, path to product
"""
TCI = read_array(band_name(
path, Bands.TCI)).transpose(1, 2, 0)
ndvi = NDVI(path)[..., None]
mask = cloud_mask(path + '/')
P, D = forest_probability(ndvi, mask, missing_values=0.5)
X = np.linspace(np.min(ndvi), np.max(ndvi), 100)
Y = (multi_normal_pdf(X, D['means'][0], D['covariances'][0]) *
D['weights'][0] +
multi_normal_pdf(X, D['means'][1], D['covariances'][1]) *
D['weights'][1])
plt.figure(figsize=(18, 5))
plt.subplot(141)
plt.imshow(TCI)
plt.subplot(142)
plt.imshow(mask)
plt.subplot(143)
plt.imshow(P, cmap='gray')
plt.subplot(144)
plt.hist(ndvi[mask == 1].flatten(), bins=100, normed=True)
plt.plot(X, Y)
plt.show()
def index_tensor(path, size=None):
"""
Calculates 13 different deforestation indices
:param path: str, path to product.
:param size: (height, width), output size.
If not given, use maximal size of bands.
:return: array (size, 13)
"""
B02 = read_array(band_name(path, Bands.B02))
if size is None:
size = B02.shape
B02 = resize_band(B02, size).astype('float32')
B03 = resize_band(read_array(band_name(path, Bands.B03)),
size).astype('float32')
B04 = resize_band(read_array(band_name(path, Bands.B04)),
size).astype('float32')
B05 = resize_band(read_array(band_name(path, Bands.B05)),
size).astype('float32')
B06 = resize_band(read_array(band_name(path, Bands.B06)),
size).astype('float32')
B08 = resize_band(read_array(band_name(path, Bands.B08)),
size).astype('float32')
B11 = resize_band(read_array(band_name(path, Bands.B11)),
size).astype('float32')
# B12 = resize_band(read_array(band_name(path, Bands.B12)),
# size).astype('float32')
# All indices take two classes corresponding to ground and forest
# all indices are written such that obtained class distribution with
# greater mean value corresponds to forest
index = [
('NDVI', (B08 - B04) / (B08 + B04)),
# ('SR', B08 / B04),
('MSI', -(B11 / B08)),
('NDVI705', (B06 - B05) / (B06 + B05)),
# ('SAVI', 1.5 * (B08 - B04) / (B08 + B04 + 0.5)),
# ('PSRI-NIR', (B04 - B02) / B08),
('PSRI', -(B04 - B02) / B05),
# ('NBR-RAW', (B08 - B12) / (B08 + B12)),
# ('MSAVI2', (B08 + 1) - 0.5 * np.sqrt((2 * B08 - 1) ** 2 + 8 * B04)),
# ('LAI-SAVI',
# -np.log(0.371 + 1.5 * (B08 - B04) / (B08 + B04 + 0.5)) / 2.4),
('GRVI1', -(B04 - B03) / (B04 + B03)),
# ('GNDVI', (B08 - B03) / (B08 + B03)),
# ('EVI2', 2.5 * (B08 - B04) / (B08 + 2.4 * B04 + 1)),
# ('NDMI', (B08 - B11) / (B11 + B08))
]
return np.stack([x[1] for x in index], axis=-1)
def align_data(data_path, aligned_data_path=None, align_info_file=None):
"""
Aligns all products in data directory by given base product.
e.g. align_data("Ijevan", "Ijevan_aligned")
:param data_path: str, path to preprocessed data / Ijevan
:param aligned_data_path: str, directory for aligned products
:param align_info_file: str, path to align_info.json, if not given,
uses file in data_path.
"""
if aligned_data_path is None:
aligned_data_path = os.path.join(data_path, 'aligned')
if align_info_file is None:
align_info_file = os.path.join(data_path, 'align_info.json')
# create directory for aligned data
os.makedirs(aligned_data_path, exist_ok=True)
with open(align_info_file, "r") as f:
align_info = json.load(f)
for product in os.listdir(data_path):
path = os.path.join(data_path, product)
product_title = get_product_title(path)
if product_title is None:
continue
warp_matrix = align_info["warp_matrices"].get(product_title)
if warp_matrix is not None:
print('Aligning {}...'.format(path))
warp_matrix = np.array(warp_matrix, dtype=np.float32)
align_product(path, warp_matrix,
os.path.join(aligned_data_path, product))
else:
copy_tree(path, os.path.join(aligned_data_path, product))
shutil.copy(
band_name(os.path.join(aligned_data_path, product), Bands.TCI),
os.path.join(aligned_data_path, product + '.tiff'))
def save_color_image(directory, r_band, g_band, b_band, suffix='TCI1',
bright_limit=3500, output_file=None):
"""
Creates color image from given bands
:param directory: str, directory, where are located band files
:param r_band: Bands enum, red band
:param g_band: Bands enum, suffix of green band
:param b_band: Bands enum, suffix of blue band
:param suffix: str, suffix for output file
:param bright_limit: Supremum of chanel brightness.
Each value in cannel array greater than bright_limit
to be assigned bright_limit
:param output_file: str
output file name. By default it saves into same folder
:return array, mixed image array
"""
channel_names = [r_band, g_band, b_band]
channels = [None, None, None]
for c in range(len(channel_names)):
file = band_name(directory, channel_names[c])
if file is None:
raise Exception('"{}" band not found in {}.'.
format(channel_names[c].value, directory))
channels[c] = read_array(file)
red_channel, green_channel, blue_channel = channels
dataset = gdal.Open(file)
if output_file is None:
output_file = os.path.splitext(file)[0][:-3] + suffix + '.tiff'
if not (red_channel.shape == green_channel.shape == blue_channel.shape):
print('Bands have different resolution.' +
str((red_channel.shape, green_channel.shape,
blue_channel.shape)))
resolution = max(red_channel.shape, green_channel.shape,
blue_channel.shape)
red_channel = resize_band(red_channel, resolution)
green_channel = resize_band(green_channel, resolution)
blue_channel = resize_band(blue_channel, resolution)
image = mix(red_channel, green_channel, blue_channel, bright_limit)
# Create gtif file
logging.debug('Creating ' + output_file)
print('Color file: ' + os.path.split(output_file)[-1])
driver = gdal.GetDriverByName("GTiff")
dst_ds = driver.Create(output_file, image.shape[1], image.shape[0],
image.shape[2], gdal.GDT_Byte)
print(output_file)
# Writing output raster
for j in range(image.shape[2]):
dst_ds.GetRasterBand(j + 1).WriteArray(image[..., j])
# Setting extension of output raster
dst_ds.SetGeoTransform(dataset.GetGeoTransform())
wkt = dataset.GetProjection()
# Setting spatial reference of output raster
srs = osr.SpatialReference()
srs.ImportFromWkt(wkt)
dst_ds.SetProjection(srs.ExportToWkt())
# Close output raster dataset
dataset = None
dst_ds = None
return image
def copy_and_format_names(origin, destination, selection=None):
"""
Remove duplicate images.
:param origin: str, path to original data.
:param destination: str, directory to copy and format files.
:param selection: array, coordinates to crop images
"""
# From products with the same date-time keep only one.
product_paths = []
dates = set()
for product in os.listdir(origin):
info_file = os.path.join(origin, product, 'info.json')
if os.path.exists(info_file) is False:
continue
info = json.load(open(info_file, 'r'))
date = timestamp_to_datetime(info['Sensing start'])
if date in dates:
continue
dates.add(date)
product_paths.append(os.path.join(origin, product))
os.makedirs(destination, exist_ok=True)
for path in product_paths:
info = json.load(open(os.path.join(path, 'info.json'), 'r'))
if info['Satellite'] != 'Sentinel-2':
continue
date_str = 'product ' + '{:%Y-%m-%d %H:%M}'.format(
timestamp_to_datetime(info['Sensing start']))
tail = None
for name in os.listdir(path):
if name.startswith('tail.'):
tail = os.path.join(path, name)
break
if tail is None:
continue
# useful bands
bands = [
Bands.RED, Bands.GREEN, Bands.BLUE, Bands.NIR, Bands.B05,
Bands.B06, Bands.B07, Bands.B01, Bands.B11, Bands.B12, Bands.B10
]
os.makedirs(os.path.join(destination, date_str), exist_ok=True)
for band in bands:
if selection is None:
shutil.copy(
band_name(tail, band),
os.path.join(destination, date_str, band.value + '.tiff'))
else:
crop(selection, band_name(tail, band),
os.path.join(destination, date_str, band.value + '.tiff'))
# create colored image
save_color_image(os.path.join(destination, date_str),
Bands.RED,
Bands.GREEN,
Bands.BLUE,
output_file=os.path.join(destination, date_str,
Bands.TCI.value + '.tiff'))
# copy info files
shutil.copy(os.path.join(path, 'info.json'),
os.path.join(destination, date_str))
shutil.copy(os.path.join(path, 'info.txt'),
os.path.join(destination, date_str))