def cloud_mask(bands_directory, cloud_threshold=1500, shadow_threshold=20, save=False): """ Get cloud mask from bands of the image in <directory> according to thresholding images by red, green and blue channels :param directory: The directory of images :param threhold: If red, green and blue values of the pixel are greater than <threshold> to be considered as cloud pixel :return: 2D array with 1s and 0s, where 0s corespond to cloudy pixels 1s - to cloud-free pixels. """ red_band_name = band_name(bands_directory, Bands.RED) green_band_name = band_name(bands_directory, Bands.GREEN) blue_band_name = band_name(bands_directory, Bands.BLUE) red_band = read_array(red_band_name) green_band = read_array(green_band_name) blue_band = read_array(blue_band_name) cloud_mask = ((red_band > cloud_threshold) * (green_band > cloud_threshold) * (blue_band > cloud_threshold)) shadow_mask = ((red_band < shadow_threshold) * (green_band < shadow_threshold) * (blue_band < shadow_threshold)) mask = ~(cloud_mask + shadow_mask) mask = mask.astype('int') if save: with open('{}/cloud_mask.pkl'.format(bands_directory), 'wb') as f: pkl.dump(mask, f) return mask
def main(args): print("Search colored image...") colored_image_name = band_name(args.product, Bands.TCI, extension='.jp2') if colored_image_name is not None: print("There is colored image") image = read_array(colored_image_name).transpose(1, 2, 0) else: print("There isn't colored image. Try mix...\n") image = save_color_image(args.product, Bands.RED, Bands.GREEN, Bands.BLUE, bright_limit=args.bright_limit, output_file=args.output) if args.plot or args.save is not None: print("\nPrepare plot...") plt.figure(figsize=args.size) plt.imshow(image) if args.plot: print("\nPlot...") plt.show() if args.save is not None: print("\nSave in {}...".format(args.save)) plt.savefig(args.save) return image
def NDVI(path, size=None): """ Calculates NDVI for product :param path: str, path to product. :param size: (height, width), output size. If not given, use size of bands. :return: array, NDVI """ NIR = read_array(band_name(path, Bands.NIR)) RED = read_array(band_name(path, Bands.RED)) if size is None: size = max(NIR.shape, RED.shape) NIR = resize_band(NIR, size) RED = resize_band(RED, size) return (NIR.astype('float') - RED.astype('float')) / ( NIR.astype('float') + RED.astype('float'))
def debug_forest_probability(path): """ Shows image, cloud mask, probabilities and NDVI distribution. :param path: str, path to product """ TCI = read_array(band_name( path, Bands.TCI)).transpose(1, 2, 0) ndvi = NDVI(path)[..., None] mask = cloud_mask(path + '/') P, D = forest_probability(ndvi, mask, missing_values=0.5) X = np.linspace(np.min(ndvi), np.max(ndvi), 100) Y = (multi_normal_pdf(X, D['means'][0], D['covariances'][0]) * D['weights'][0] + multi_normal_pdf(X, D['means'][1], D['covariances'][1]) * D['weights'][1]) plt.figure(figsize=(18, 5)) plt.subplot(141) plt.imshow(TCI) plt.subplot(142) plt.imshow(mask) plt.subplot(143) plt.imshow(P, cmap='gray') plt.subplot(144) plt.hist(ndvi[mask == 1].flatten(), bins=100, normed=True) plt.plot(X, Y) plt.show()
def search_by_vectors(self, vectors): vectors = read_array(vectors, SIFT_DIMENSIONS) # ====== trick code start =========== count = vectors.shape[0] vectors = np.vstack((vectors, vectors)) vectors = vectors[0:count, :] # ====== trick code end =========== ids = [None] results = self.search(ids, [vectors]) return results
def index_tensor(path, size=None): """ Calculates 13 different deforestation indices :param path: str, path to product. :param size: (height, width), output size. If not given, use maximal size of bands. :return: array (size, 13) """ B02 = read_array(band_name(path, Bands.B02)) if size is None: size = B02.shape B02 = resize_band(B02, size).astype('float32') B03 = resize_band(read_array(band_name(path, Bands.B03)), size).astype('float32') B04 = resize_band(read_array(band_name(path, Bands.B04)), size).astype('float32') B05 = resize_band(read_array(band_name(path, Bands.B05)), size).astype('float32') B06 = resize_band(read_array(band_name(path, Bands.B06)), size).astype('float32') B08 = resize_band(read_array(band_name(path, Bands.B08)), size).astype('float32') B11 = resize_band(read_array(band_name(path, Bands.B11)), size).astype('float32') # B12 = resize_band(read_array(band_name(path, Bands.B12)), # size).astype('float32') # All indices take two classes corresponding to ground and forest # all indices are written such that obtained class distribution with # greater mean value corresponds to forest index = [ ('NDVI', (B08 - B04) / (B08 + B04)), # ('SR', B08 / B04), ('MSI', -(B11 / B08)), ('NDVI705', (B06 - B05) / (B06 + B05)), # ('SAVI', 1.5 * (B08 - B04) / (B08 + B04 + 0.5)), # ('PSRI-NIR', (B04 - B02) / B08), ('PSRI', -(B04 - B02) / B05), # ('NBR-RAW', (B08 - B12) / (B08 + B12)), # ('MSAVI2', (B08 + 1) - 0.5 * np.sqrt((2 * B08 - 1) ** 2 + 8 * B04)), # ('LAI-SAVI', # -np.log(0.371 + 1.5 * (B08 - B04) / (B08 + B04 + 0.5)) / 2.4), ('GRVI1', -(B04 - B03) / (B04 + B03)), # ('GNDVI', (B08 - B03) / (B08 + B03)), # ('EVI2', 2.5 * (B08 - B04) / (B08 + 2.4 * B04 + 1)), # ('NDMI', (B08 - B11) / (B11 + B08)) ] return np.stack([x[1] for x in index], axis=-1)
def save_color_image(directory, r_band, g_band, b_band, suffix='TCI1', bright_limit=3500, output_file=None): """ Creates color image from given bands :param directory: str, directory, where are located band files :param r_band: Bands enum, red band :param g_band: Bands enum, suffix of green band :param b_band: Bands enum, suffix of blue band :param suffix: str, suffix for output file :param bright_limit: Supremum of chanel brightness. Each value in cannel array greater than bright_limit to be assigned bright_limit :param output_file: str output file name. By default it saves into same folder :return array, mixed image array """ channel_names = [r_band, g_band, b_band] channels = [None, None, None] for c in range(len(channel_names)): file = band_name(directory, channel_names[c]) if file is None: raise Exception('"{}" band not found in {}.'. format(channel_names[c].value, directory)) channels[c] = read_array(file) red_channel, green_channel, blue_channel = channels dataset = gdal.Open(file) if output_file is None: output_file = os.path.splitext(file)[0][:-3] + suffix + '.tiff' if not (red_channel.shape == green_channel.shape == blue_channel.shape): print('Bands have different resolution.' + str((red_channel.shape, green_channel.shape, blue_channel.shape))) resolution = max(red_channel.shape, green_channel.shape, blue_channel.shape) red_channel = resize_band(red_channel, resolution) green_channel = resize_band(green_channel, resolution) blue_channel = resize_band(blue_channel, resolution) image = mix(red_channel, green_channel, blue_channel, bright_limit) # Create gtif file logging.debug('Creating ' + output_file) print('Color file: ' + os.path.split(output_file)[-1]) driver = gdal.GetDriverByName("GTiff") dst_ds = driver.Create(output_file, image.shape[1], image.shape[0], image.shape[2], gdal.GDT_Byte) print(output_file) # Writing output raster for j in range(image.shape[2]): dst_ds.GetRasterBand(j + 1).WriteArray(image[..., j]) # Setting extension of output raster dst_ds.SetGeoTransform(dataset.GetGeoTransform()) wkt = dataset.GetProjection() # Setting spatial reference of output raster srs = osr.SpatialReference() srs.ImportFromWkt(wkt) dst_ds.SetProjection(srs.ExportToWkt()) # Close output raster dataset dataset = None dst_ds = None return image
# In the debug mode, instead of sending replies # with praw, it just prints everything on tty debug = 0 # Dictionary of all the investors users = utils.read_investors(data_folder + investors_file) print(users) # Array to store done and awaiting investments done = utils.read_investments(data_folder + done_file) awaiting = utils.read_investments(data_folder + awaiting_file) # Array to store IDs of parsed comment, so we won't parse the same # comment multiple times checked_comments = utils.read_array(data_folder + checked_file) def save_data(): global users, awaiting, done utils.write_investors(data_folder + investors_file, users) utils.write_investments(data_folder + awaiting_file, awaiting) utils.write_investments(data_folder + done_file, done) utils.write_array(data_folder + checked_file, checked_comments) def send_not(comment, string, save): try: if (save): save_data() global debug commentID = 0 if (debug):
workers=multiprocessing.cpu_count(), model_path=args.wm) utils.logger.info("word2vec training finished") elif args.train_cnn or args.predict_cnn or args.validate_cnn: train_data = None test_data = None if args.train_cnn or args.validate_cnn: loaded = False train_labels = utils.read_labels(args.l) utils.logger.info("labels reading finished") if args.use_d2v and utils.utils.is_path_accessible(args.dd): utils.logger.info( "try to load from doc2vec train data from specified path") train_data = utils.read_array(args.dd) utils.logger.info("load doc2vec train data successfully") loaded = True elif args.use_w2v and utils.is_path_accessible(args.wd): utils.logger.info( "try to load from word2vec train data from specified path") train_data = utils.read_array(args.wd) utils.logger.info("load word2vec train data successfully") loaded = True if not loaded: train_text = utils.read_text(args.d) utils.logger.info("train text reading finished") if args.use_d2v: train_tokens = utils.tokenize_paragraph_d2v(train_text)
def plot_deforestation_ponts(df, image, save_dir=None, show=False, c=0.6, plot=True): """ Plot probability map, high probability points on real image, scatter and 2D histogram of pixel-probabilities :param df: Dataframe. You can get it by df = forest_probabilities_TS(path) :param image: Colored image to visually check our result :param save_dir: Where you'd like to save plots :param show: Do you want to show plots or not :param c: Threshold of probabilities :param plot: Plot or not :return: Dataframe of probabilities """ start2016, end2016 = datetime(2016, 1, 1), datetime(2016, 12, 31) start2017, end2017 = datetime(2017, 1, 1), datetime(2017, 12, 31) s2016 = df[(df.index > start2016) & (df.index < end2016)].mean(axis=0) s2017 = df[(df.index > start2017) & (df.index < end2017)].mean(axis=0) ss = pd.DataFrame([s2016, s2017], index=[2016, 2017]).T prob_mask = ss[2016] * (1 - ss[2017]) deforestation_points = ss.dropna().index[(prob_mask.dropna() > c)] print('Points: {}'.format(len(deforestation_points))) image = read_array(image).transpose(1, 2, 0) if plot: pred = prob_mask.values.reshape(image.shape[0], image.shape[1]) # plot probability mask plt.figure(figsize=(10, 4.1)) plt.subplot(122) plt.axis("off") plt.imshow(image) plt.scatter([y for x, y in deforestation_points], [x for x, y in deforestation_points], s=0.5, c="Orange", lw=0) plt.suptitle("Deforestation Probability", fontsize=20) plt.subplot(121) plt.axis("off") plt.imshow(pred, cmap="hot_r") plt.colorbar(fraction=0.037, pad=0.04) if save_dir is not None: plt.savefig(os.path.join(save_dir, "deforestatin.png"), bbox_inches='tight', dpi=300) # plot forest probabilities plt.figure(figsize=(10, 4.5)) y = np.linspace(0, 1 - c, 100) x = c / (1 - y) plt.subplot(121) plt.margins(0) plt.plot(x, y, color="red") plt.scatter(ss[2016], ss[2017], s=1.5, lw=0, alpha=0.8) plt.suptitle("Forest Probabilities of pixels", fontsize=20) plt.subplot(122) plt.axis("off") plt.hist2d(ss.dropna()[2016], ss.dropna()[2017], norm=LogNorm(), bins=12) plt.colorbar(fraction=0.037, pad=0.04) if save_dir is not None: plt.savefig(os.path.join(save_dir, "scatter.png"), bbox_inches='tight', dpi=300) if show: plt.show() return ss
def cluster_images(in_path,out_path): images=utils.read_array(in_path) clusters=clustering_mini_batch(images) utils.save_object(out_path,clusters)