def cloud_mask(bands_directory, cloud_threshold=1500,
shadow_threshold=20, save=False):
"""
Get cloud mask from bands of the image in <directory>
according to thresholding images by red, green and blue channels
:param directory: The directory of images
:param threhold: If red, green and blue values of the pixel
are greater than <threshold> to be considered
as cloud pixel
:return: 2D array with 1s and 0s, where 0s corespond to cloudy pixels
1s - to cloud-free pixels.
"""
red_band_name = band_name(bands_directory, Bands.RED)
green_band_name = band_name(bands_directory, Bands.GREEN)
blue_band_name = band_name(bands_directory, Bands.BLUE)
red_band = read_array(red_band_name)
green_band = read_array(green_band_name)
blue_band = read_array(blue_band_name)
cloud_mask = ((red_band > cloud_threshold) * (green_band > cloud_threshold)
* (blue_band > cloud_threshold))
shadow_mask = ((red_band < shadow_threshold)
* (green_band < shadow_threshold)
* (blue_band < shadow_threshold))
mask = ~(cloud_mask + shadow_mask)
mask = mask.astype('int')
if save:
with open('{}/cloud_mask.pkl'.format(bands_directory), 'wb') as f:
pkl.dump(mask, f)
return mask
def main(args):
print("Search colored image...")
colored_image_name = band_name(args.product, Bands.TCI, extension='.jp2')
if colored_image_name is not None:
print("There is colored image")
image = read_array(colored_image_name).transpose(1, 2, 0)
else:
print("There isn't colored image. Try mix...\n")
image = save_color_image(args.product, Bands.RED,
Bands.GREEN, Bands.BLUE,
bright_limit=args.bright_limit,
output_file=args.output)
if args.plot or args.save is not None:
print("\nPrepare plot...")
plt.figure(figsize=args.size)
plt.imshow(image)
if args.plot:
print("\nPlot...")
plt.show()
if args.save is not None:
print("\nSave in {}...".format(args.save))
plt.savefig(args.save)
return image
def NDVI(path, size=None):
"""
Calculates NDVI for product
:param path: str, path to product.
:param size: (height, width), output size.
If not given, use size of bands.
:return: array, NDVI
"""
NIR = read_array(band_name(path, Bands.NIR))
RED = read_array(band_name(path, Bands.RED))
if size is None:
size = max(NIR.shape, RED.shape)
NIR = resize_band(NIR, size)
RED = resize_band(RED, size)
return (NIR.astype('float') - RED.astype('float')) / (
NIR.astype('float') + RED.astype('float'))
def debug_forest_probability(path):
"""
Shows image, cloud mask, probabilities and NDVI distribution.
:param path: str, path to product
"""
TCI = read_array(band_name(
path, Bands.TCI)).transpose(1, 2, 0)
ndvi = NDVI(path)[..., None]
mask = cloud_mask(path + '/')
P, D = forest_probability(ndvi, mask, missing_values=0.5)
X = np.linspace(np.min(ndvi), np.max(ndvi), 100)
Y = (multi_normal_pdf(X, D['means'][0], D['covariances'][0]) *
D['weights'][0] +
multi_normal_pdf(X, D['means'][1], D['covariances'][1]) *
D['weights'][1])
plt.figure(figsize=(18, 5))
plt.subplot(141)
plt.imshow(TCI)
plt.subplot(142)
plt.imshow(mask)
plt.subplot(143)
plt.imshow(P, cmap='gray')
plt.subplot(144)
plt.hist(ndvi[mask == 1].flatten(), bins=100, normed=True)
plt.plot(X, Y)
plt.show()
def search_by_vectors(self, vectors):
vectors = read_array(vectors, SIFT_DIMENSIONS)
# ====== trick code start ===========
count = vectors.shape[0]
vectors = np.vstack((vectors, vectors))
vectors = vectors[0:count, :]
# ====== trick code end ===========
ids = [None]
results = self.search(ids, [vectors])
return results
def index_tensor(path, size=None):
"""
Calculates 13 different deforestation indices
:param path: str, path to product.
:param size: (height, width), output size.
If not given, use maximal size of bands.
:return: array (size, 13)
"""
B02 = read_array(band_name(path, Bands.B02))
if size is None:
size = B02.shape
B02 = resize_band(B02, size).astype('float32')
B03 = resize_band(read_array(band_name(path, Bands.B03)),
size).astype('float32')
B04 = resize_band(read_array(band_name(path, Bands.B04)),
size).astype('float32')
B05 = resize_band(read_array(band_name(path, Bands.B05)),
size).astype('float32')
B06 = resize_band(read_array(band_name(path, Bands.B06)),
size).astype('float32')
B08 = resize_band(read_array(band_name(path, Bands.B08)),
size).astype('float32')
B11 = resize_band(read_array(band_name(path, Bands.B11)),
size).astype('float32')
# B12 = resize_band(read_array(band_name(path, Bands.B12)),
# size).astype('float32')
# All indices take two classes corresponding to ground and forest
# all indices are written such that obtained class distribution with
# greater mean value corresponds to forest
index = [
('NDVI', (B08 - B04) / (B08 + B04)),
# ('SR', B08 / B04),
('MSI', -(B11 / B08)),
('NDVI705', (B06 - B05) / (B06 + B05)),
# ('SAVI', 1.5 * (B08 - B04) / (B08 + B04 + 0.5)),
# ('PSRI-NIR', (B04 - B02) / B08),
('PSRI', -(B04 - B02) / B05),
# ('NBR-RAW', (B08 - B12) / (B08 + B12)),
# ('MSAVI2', (B08 + 1) - 0.5 * np.sqrt((2 * B08 - 1) ** 2 + 8 * B04)),
# ('LAI-SAVI',
# -np.log(0.371 + 1.5 * (B08 - B04) / (B08 + B04 + 0.5)) / 2.4),
('GRVI1', -(B04 - B03) / (B04 + B03)),
# ('GNDVI', (B08 - B03) / (B08 + B03)),
# ('EVI2', 2.5 * (B08 - B04) / (B08 + 2.4 * B04 + 1)),
# ('NDMI', (B08 - B11) / (B11 + B08))
]
return np.stack([x[1] for x in index], axis=-1)
def save_color_image(directory, r_band, g_band, b_band, suffix='TCI1',
bright_limit=3500, output_file=None):
"""
Creates color image from given bands
:param directory: str, directory, where are located band files
:param r_band: Bands enum, red band
:param g_band: Bands enum, suffix of green band
:param b_band: Bands enum, suffix of blue band
:param suffix: str, suffix for output file
:param bright_limit: Supremum of chanel brightness.
Each value in cannel array greater than bright_limit
to be assigned bright_limit
:param output_file: str
output file name. By default it saves into same folder
:return array, mixed image array
"""
channel_names = [r_band, g_band, b_band]
channels = [None, None, None]
for c in range(len(channel_names)):
file = band_name(directory, channel_names[c])
if file is None:
raise Exception('"{}" band not found in {}.'.
format(channel_names[c].value, directory))
channels[c] = read_array(file)
red_channel, green_channel, blue_channel = channels
dataset = gdal.Open(file)
if output_file is None:
output_file = os.path.splitext(file)[0][:-3] + suffix + '.tiff'
if not (red_channel.shape == green_channel.shape == blue_channel.shape):
print('Bands have different resolution.' +
str((red_channel.shape, green_channel.shape,
blue_channel.shape)))
resolution = max(red_channel.shape, green_channel.shape,
blue_channel.shape)
red_channel = resize_band(red_channel, resolution)
green_channel = resize_band(green_channel, resolution)
blue_channel = resize_band(blue_channel, resolution)
image = mix(red_channel, green_channel, blue_channel, bright_limit)
# Create gtif file
logging.debug('Creating ' + output_file)
print('Color file: ' + os.path.split(output_file)[-1])
driver = gdal.GetDriverByName("GTiff")
dst_ds = driver.Create(output_file, image.shape[1], image.shape[0],
image.shape[2], gdal.GDT_Byte)
print(output_file)
# Writing output raster
for j in range(image.shape[2]):
dst_ds.GetRasterBand(j + 1).WriteArray(image[..., j])
# Setting extension of output raster
dst_ds.SetGeoTransform(dataset.GetGeoTransform())
wkt = dataset.GetProjection()
# Setting spatial reference of output raster
srs = osr.SpatialReference()
srs.ImportFromWkt(wkt)
dst_ds.SetProjection(srs.ExportToWkt())
# Close output raster dataset
dataset = None
dst_ds = None
return image
# In the debug mode, instead of sending replies
# with praw, it just prints everything on tty
debug = 0
# Dictionary of all the investors
users = utils.read_investors(data_folder + investors_file)
print(users)
# Array to store done and awaiting investments
done = utils.read_investments(data_folder + done_file)
awaiting = utils.read_investments(data_folder + awaiting_file)
# Array to store IDs of parsed comment, so we won't parse the same
# comment multiple times
checked_comments = utils.read_array(data_folder + checked_file)
def save_data():
global users, awaiting, done
utils.write_investors(data_folder + investors_file, users)
utils.write_investments(data_folder + awaiting_file, awaiting)
utils.write_investments(data_folder + done_file, done)
utils.write_array(data_folder + checked_file, checked_comments)
def send_not(comment, string, save):
try:
if (save):
save_data()
global debug
commentID = 0
if (debug):
workers=multiprocessing.cpu_count(),
model_path=args.wm)
utils.logger.info("word2vec training finished")
elif args.train_cnn or args.predict_cnn or args.validate_cnn:
train_data = None
test_data = None
if args.train_cnn or args.validate_cnn:
loaded = False
train_labels = utils.read_labels(args.l)
utils.logger.info("labels reading finished")
if args.use_d2v and utils.utils.is_path_accessible(args.dd):
utils.logger.info(
"try to load from doc2vec train data from specified path")
train_data = utils.read_array(args.dd)
utils.logger.info("load doc2vec train data successfully")
loaded = True
elif args.use_w2v and utils.is_path_accessible(args.wd):
utils.logger.info(
"try to load from word2vec train data from specified path")
train_data = utils.read_array(args.wd)
utils.logger.info("load word2vec train data successfully")
loaded = True
if not loaded:
train_text = utils.read_text(args.d)
utils.logger.info("train text reading finished")
if args.use_d2v:
train_tokens = utils.tokenize_paragraph_d2v(train_text)
def plot_deforestation_ponts(df,
image,
save_dir=None,
show=False,
c=0.6,
plot=True):
"""
Plot probability map, high probability points on real image, scatter and 2D
histogram of pixel-probabilities
:param df: Dataframe. You can get it by df = forest_probabilities_TS(path)
:param image: Colored image to visually check our result
:param save_dir: Where you'd like to save plots
:param show: Do you want to show plots or not
:param c: Threshold of probabilities
:param plot: Plot or not
:return: Dataframe of probabilities
"""
start2016, end2016 = datetime(2016, 1, 1), datetime(2016, 12, 31)
start2017, end2017 = datetime(2017, 1, 1), datetime(2017, 12, 31)
s2016 = df[(df.index > start2016) & (df.index < end2016)].mean(axis=0)
s2017 = df[(df.index > start2017) & (df.index < end2017)].mean(axis=0)
ss = pd.DataFrame([s2016, s2017], index=[2016, 2017]).T
prob_mask = ss[2016] * (1 - ss[2017])
deforestation_points = ss.dropna().index[(prob_mask.dropna() > c)]
print('Points: {}'.format(len(deforestation_points)))
image = read_array(image).transpose(1, 2, 0)
if plot:
pred = prob_mask.values.reshape(image.shape[0], image.shape[1])
# plot probability mask
plt.figure(figsize=(10, 4.1))
plt.subplot(122)
plt.axis("off")
plt.imshow(image)
plt.scatter([y for x, y in deforestation_points],
[x for x, y in deforestation_points],
s=0.5,
c="Orange",
lw=0)
plt.suptitle("Deforestation Probability", fontsize=20)
plt.subplot(121)
plt.axis("off")
plt.imshow(pred, cmap="hot_r")
plt.colorbar(fraction=0.037, pad=0.04)
if save_dir is not None:
plt.savefig(os.path.join(save_dir, "deforestatin.png"),
bbox_inches='tight',
dpi=300)
# plot forest probabilities
plt.figure(figsize=(10, 4.5))
y = np.linspace(0, 1 - c, 100)
x = c / (1 - y)
plt.subplot(121)
plt.margins(0)
plt.plot(x, y, color="red")
plt.scatter(ss[2016], ss[2017], s=1.5, lw=0, alpha=0.8)
plt.suptitle("Forest Probabilities of pixels", fontsize=20)
plt.subplot(122)
plt.axis("off")
plt.hist2d(ss.dropna()[2016],
ss.dropna()[2017],
norm=LogNorm(),
bins=12)
plt.colorbar(fraction=0.037, pad=0.04)
if save_dir is not None:
plt.savefig(os.path.join(save_dir, "scatter.png"),
bbox_inches='tight',
dpi=300)
if show:
plt.show()
return ss
def cluster_images(in_path,out_path):
images=utils.read_array(in_path)
clusters=clustering_mini_batch(images)
utils.save_object(out_path,clusters)
