def overlap_mask_rasters(in_folder1,
in_folder2,
out_folder,
pattern=None,
erase_thresh=150):
'''Adds the values of raster maps together in areas where there is overlap.
Can be used to display results from two different classifications in one
single image raster. Areas of agreement between two masks will be given
a mask value of 5, while areas where only method 1 (in_folder1) classified
will be given a value of 2, and areas where only method 2 (in_folder2)
classified will be given a value of 3. Can then use this layer in ArcMap
or similar to visualize the results.
Parameters
----------
in_folder1: str
Folder where mask images from method one are located.
in_folder2: str
Folder where mask images from method two are located.
out_folder: str
Folder to save the output mask files to.
pattern: str (optional)
A string defining a keyword or pattern to identify target mask images
from the input folders.
erase_thresh: int
Value determining the minimum threshold for masks. If a single continuous
cluster of pixels which make up an object mask in an image are less than
the threshold, then this part of the mask will be removed. Useful for
removing small noisy detections.
'''
files_mask = sorted(getFiles(in_folder1, ending='.png', pattern=pattern))
files_deep = sorted(getFiles(in_folder2, ending='.png', pattern=pattern))
for fm, fd in zip(files_mask, files_deep):
assert fm.rsplit('\\', 1)[1] == fd.rsplit('\\',
1)[1], 'Files dont match'
print(fm.rsplit('\\', 1)[1])
with rasterio.open(fm) as src:
mask = src.read(1)
mask = erase(mask, erase_thresh)
with rasterio.open(fd) as src:
deep = src.read(1)
deep = erase(deep, erase_thresh)
# pixels with value of 2 are from mask, with 3 are from deep, with 5 are from both agreeing
# Will use this to symbolize the images in arcmap.
mask[mask != 0] = 2
deep[deep != 0] = 3
out = mask + deep
out = np.expand_dims(out, axis=0)
outfile = os.path.join(out_folder, fm.rsplit('\\', 1)[1])
with rasterio.open(outfile, 'w', **src.meta) as dst:
dst.write(out)
def match_files(dir_to_match_with,
dir_to_move_from,
dir_to_move_to,
copy=False):
'''Moves (or copies) only files which have the same name as those in another
folder. Useful when one has a folder of mask files which have been split into
training and validation sets, and one wishes to then move the same image
files so that both mask and images folders have the same matching files.
Parameters
----------
dir_to_match_with: str
Path of the directory containing the list of file names you wish to
also have in a seperate directory
dir_to_move_from: str
Path of the directory containing the files you wish to extract.
dir_to_move_to: str
Path of the directory where files should be moved into.
'''
files_match = os.listdir(dir_to_match_with)
files_to_move = getFiles(dir_to_move_from, ending='.png')
for file in files_to_move:
i = file.rsplit('\\', 1)[1]
if i in files_match:
if not copy:
shutil.move(file, dir_to_move_to)
if copy:
shutil.copy(file, dir_to_move_to)
def check_hedge_sizes(mask_folder, img_folder):
mask_files = getFiles(mask_folder, ending='.png')
for i, file in enumerate(mask_files):
img_id = file.rsplit('\\', 1)[1]
i_id = str(re.findall(r'\d+',
img_id)).strip('[]').replace("'", "").replace(
",", "").replace(" ", "_")
with rasterio.open(file) as src:
w = src.read(1)
if np.max(w) != 1:
print('no hedge')
print('DELETING', file)
os.remove(file)
file2 = os.path.join(img_folder, img_id)
print('DELETING', file2)
os.remove(file2)
continue
m_info = create_annotation_info(w, i_id, i, file, (320, 320))
areas = []
for entry in m_info:
areas.append(entry['area'])
if all(area < 20 for area in areas):
print('Hedges too small')
print('DELETING', file)
os.remove(file)
file2 = os.path.join(img_folder, img_id)
print('DELETING', file2)
os.remove(file2)
def __init__(self, directory, mask=None, ending='.tif', pattern=None):
self.dir = directory
self.raster_paths = getFiles(directory, ending=ending, pattern=pattern)
self.rasters = [
rasterio.open(_raster) for _raster in self.raster_paths
]
self.meta = [rast.meta.copy() for rast in self.rasters]
if mask is not None:
self.mask = fiona.open(mask, "r")
def mosaic_rasters(in_folder,
out_folder,
mosaic_name='mosaic',
pattern=None,
clean=False):
'''Mosaics a batch of image tiles into one single continuous image.
Parameters
----------
in_folder: str
Folder where images to be mosaiced are located.
out_folder: str
Folder where mosaic file should be saved to.
mosaic_name: str (optional)
Name to be given to the mosaic files.
clean: bool
If true, will delete all individual image tiles from the in_folder.
Note
----
Raster files must have coordinate information for this method to work.
'''
mosaic = []
img_tiles = getFiles(in_folder, ending='.png', pattern=pattern)
for file in img_tiles:
src = rasterio.open(file)
mosaic.append(src)
meta_d = mosaic[0].meta.copy()
arr, xform = merge(mosaic)
print(arr.shape)
meta_d.update({
"driver": "PNG",
"height": arr.shape[1],
"width": arr.shape[2],
"count": 1,
"transform": xform,
"dtype": 'uint8'
})
if not os.path.exists(out_folder):
os.mkdir(out_folder)
print('Creating {}'.format(out_folder))
with rasterio.open(os.path.join(out_folder, mosaic_name + '.png'), 'w',
**meta_d) as dest:
dest.write_band(1, np.squeeze(arr.astype(np.uint8)))
if clean == True:
del src
del mosaic
for file in img_tiles:
os.remove(file)
os.remove(file + '.aux.xml')
def find_hedges(in_path, outpath, move=None, erase_thres=200):
"""
Finds all mask images where appropriately sized hedges are present. Deletes
mask files without hedges and moves those with hedges to a new folder.
Parameters
----------
inpath: str
Path to mask files
outpath: str
Path where masks containing hedges should be moved to.
move: str (optional)
String giving the folder path of where to move masks without hedges.
If left as none then masks without hedges will simply be deleted.
When first testing different erase thresholds this is not recommended.
More CAUTIOUS approach is to first move the masks and inspect that
only unwanted masks have been moved, and then deleting them manually.
"""
#get all mask image tiles
files = getFiles(in_path, ending='.png')
for file in files:
name = file.rsplit('\\', 1)[1]
#load hedge mask
mask = cv2.imread(file, cv2.IMREAD_GRAYSCALE)
#erase any hedge mask segments that are too small
mask = erase(mask, erase_thres)
#check if any hedges are present within the mask image
if np.max(mask) < 1:
print('removing {}'.format(file))
if move:
shutil.move(file, move)
else:
shutil.remove(file)
else:
cv2.imwrite(os.path.join(outpath, name), mask)
# -*- coding: utf-8 -*-
"""
Created on Thu Nov 28 12:19:45 2019
@author: ahls_st
"""
from filegrabber import getFiles
import rasterio
from rasterio.merge import merge
import os
import numpy as np
mosaic = []
files = getFiles(r'H:\mosaics', ending='.png')
print(files[0])
for file in files:
mos = rasterio.open(file)
mosaic.append(mos)
print(len(mosaic))
#mosaic the mask tiles into one image
meta_d = mosaic[0].meta.copy()
arr, xform = merge(mosaic)
print(arr.shape)
meta_d.update({
"driver": "PNG",
"height": arr.shape[1],
"width": arr.shape[2],
if not os.path.exists(os.path.join(aug_img_dir, 'Cleaned')):
print('Creating {}'.format(os.path.join(aug_img_dir, 'Cleaned')))
os.mkdir(os.path.join(aug_img_dir, 'Cleaned'))
# Match the training images with the mask files
# since some mask files removed we also remove the coresponding training images
match_files(os.path.join(aug_mask_dir, 'Cleaned'), aug_img_dir,
os.path.join(aug_img_dir, 'Cleaned'))
# annotate the full training and validation dataset for use in Mask R-CNN
# First get original mask file names
train = in_mask
val = r'C:\Users\ahls_st\Documents\MasterThesis\IKONOS\With_Hedges\ThreeBands\Splits\BGR\Masks\val' #change
# combine original files with augmented files
files = getFiles(train, ending='.png')
files2 = getFiles(val, ending='.png')
files3 = getFiles(os.path.join(aug_mask_dir, 'Cleaned'), ending='.png')
files = sorted(files + files2 + files3)
len(files)
# Get mask annotations info
mask_anno = []
for i, file in enumerate(files):
img_id = file.rsplit('\\', 1)[1]
#Change to your own needs if needed.
# Important is only that ID between mask and image match, format of the ID is up to the user.
i_id = str(re.findall(r'\d+',
img_id)).strip('[]').replace("'", "").replace(
",", "").replace(" ", "_")
with rasterio.open(file) as src:
print('Hedges too small')
print('DELETING', file)
os.remove(file)
file2 = os.path.join(img_folder, img_id)
print('DELETING', file2)
os.remove(file2)
if __name__ == '__main__':
#Get hedge mask images
train = r'C:\Users\ahls_st\Documents\MasterThesis\IKONOS\With_Hedges\ThreeBands\Splits\BGR\Masks\train' #change
aug = r'C:\Users\ahls_st\Documents\MasterThesis\IKONOS\With_Hedges\ThreeBands\Splits\BGR\Augs\mask\Scale' #change
val = r'C:\Users\ahls_st\Documents\MasterThesis\IKONOS\With_Hedges\ThreeBands\Splits\BGR\Masks\val' #change
files = getFiles(
train, ending='.png') #change **** ONLY IF files are not in png format
files2 = getFiles(
val, ending='.png') #change **** ONLY IF files are not in png format
files3 = getFiles(
aug, ending='.png') #change **** ONLY IF files are not in png format
files = sorted(files + files2 + files3)
len(files)
#Get mask annotations info
mask_anno = []
for i, file in enumerate(files):
img_id = file.rsplit('\\', 1)[1]
i_id = str(re.findall(r'\d+',
img_id)).strip('[]').replace("'", "").replace(
",", "").replace(" ", "_")
with rasterio.open(file) as src:
super_threshold_indices = image > 1
image[super_threshold_indices] = 1
mask = image == 1
results = ({'properties': {'raster_val': v}, 'geometry': s} for i, (s, v) in
enumerate(shapes(image, mask=mask, transform=src.meta['transform'])))
with fiona.open(
outfile, 'w',
driver="ESRI Shapefile",
crs=src.crs,
schema={'properties': [('raster_val', 'int')],
'geometry': 'Polygon'}) as dst:
dst.writerecords(results)
if __name__ == '__main__':
directory = r'H:\predictions_fully_trained_geo'
if not os.path.exists(os.path.join(directory, 'polygons')):
os.mkdir(os.path.join(directory, 'polygons'))
print('Creating {}'.format(os.path.join(directory, 'polygons')))
files = getFiles(directory, ending = '.png')
for file in files:
outpath = os.path.join(directory, 'polygons')
i = file.rsplit('\\', 1)[1]
i = os.path.splitext(i)[0]
outfile = os.path.join(outpath, i + '_polygon.shp')
shapefile_from_raster(file, outfile)
print(outfile)