def valid_mask(frame_name, mask_dict, is_prediction_data=False) -> bool:
frame_name = f"{frame_name}.tif"
target_mask_file = mask_dict.get(frame_name)
if target_mask_file == None:
return False
output = []
target_mask_path = os.path.join(os.getcwd(), "prep_tiles", "mask_tiles",
target_mask_file)
try:
with gdal_open(target_mask_path) as f:
output = f.ReadAsArray()
except FileNotFoundError:
sys.exit(f"File {target_mask_file} not found!")
output_arr = np.asarray(output).flatten()
image_size = len(output_arr)
unique_classes, unique_count = np.unique(output_arr, return_counts=True)
class_dict = dict(zip(unique_classes, unique_count))
frequency_dict = find_frequencies(class_dict, unique_classes, image_size)
if is_prediction_data:
return True
if frequency_dict.get("0") == None:
return True
# ignore cropmask files that are primarily one class (background, qgis no-data value)
if frequency_dict.get("0") < 0.8:
return True
# return True
return False
def compare_pred_mask_acc(pred_mask_pairs: Tuple[str, str]):
prediction_fpath = pred_mask_pairs[0]
ground_truth_fpath = pred_mask_pairs[1]
prediction = []
ground_truth = []
try:
with gdal_open(prediction_fpath) as f:
prediction = f.ReadAsArray()
except FileNotFoundError:
sys.exit(f"File {target_mask_file} not found!")
try:
with gdal_open(ground_truth_fpath) as f:
ground_truth = f.ReadAsArray()
except FileNotFoundError:
sys.exit(f"File {target_mask_file} not found!")
prediction = np.array(prediction).astype('uint8').reshape(1024, 1024, 1)
ground_truth = np.array(ground_truth).astype('uint8').reshape(
1024, 1024, 1).flatten()
output = np.zeros((1024, 1024, 1)).astype('float32').flatten()
for i, val in enumerate(prediction.flatten()):
# if the model predicted a crop area accurate
if ground_truth[i] == val:
output[i] = 1
# If the model predicted a crop when there was none there
elif ground_truth[i] != val and val == 1:
output[i] = 255
elif ground_truth[i] != val:
output[i] = 128
vals = np.unique(output)
total_pixels = 1024.0**2
freq_dict = {}
for val in vals:
freq_dict[val] = np.count_nonzero(output == val) / total_pixels
output = output.reshape(1024, 1024, 1)
# print(freq_dict)
return output, freq_dict
def save_img(file_path: str,
sample_file_path: str,
pred,
dem=NETWORK_DEMS) -> None:
with gdal_open(sample_file_path) as f:
mask_projection = f.GetProjection()
mask_geo_transform = f.GetGeoTransform()
write_mask_to_file(pred.reshape(dem, dem), file_path, mask_projection,
mask_geo_transform)
def read_file(file_name):
target_file_path = os.path.join(os.getcwd(), "prep_tiles", file_name)
output = []
try:
with gdal_open(target_file_path) as f:
output = f.ReadAsArray()
except FileNotFoundError:
sys.exit(f"File {target_file_path} not found!")
return output
def __load_vh_vv(self, vh_dataset_path: str, vv_dataset_path: str):
try:
with gdal_open(
os.path.join(self.dataset_directory,
vh_dataset_path)) as f:
vh = f.ReadAsArray()
except FileNotFoundError:
print(
f"Missing file {os.path.join(self.dataset_directory,vh_dataset_path)}"
)
# continue
try:
with gdal_open(
os.path.join(self.dataset_directory,
vv_dataset_path)) as f:
vv = f.ReadAsArray()
except FileNotFoundError:
print(
f"Missing file {os.path.join(self.dataset_directory,vv_dataset_path)}"
)
# continue
return vh, vv
def __get_mask(self,
sample_subset_prefix: str,
frame_number,
mask_shape=(NETWORK_DEMS, NETWORK_DEMS, 1)):
# get corresponding mask prefix (ie: CA_2019, and grab the corresponding mask file)
subset_name = f"{'_'.join(sample_subset_prefix.split('_')[:-1])}"
subset_mask_dir_name = f"{subset_name}_masks"
file_name = f"CDL_{subset_name}_mask_{frame_number}.tif"
target_folder = "masks"
mask_path = os.path.join(self.dataset_directory, target_folder,
subset_mask_dir_name, file_name)
mask = np.zeros(mask_shape).astype('float32')
try:
with gdal_open(mask_path) as f:
mask = np.array(f.ReadAsArray()).astype('float32')
except FileNotFoundError:
print(f"Mask {mask_path} missing")
return mask
def groom_imgs(directory: str) -> None:
if not check_dependencies(('gdal', 'pyplot', 'np')):
return
VH_REGEX = re.compile(r"(.*)\.vh\.tif")
f_path = os.path.join(config.DATASETS_DIR, directory)
g_path = os.path.join(config.DATASETS_DIR, f'{directory}Groomed')
done = False
count = 0
update_count = 0
for root, directories, files in os.walk(f_path):
for vh in files:
if done:
break
m = re.match(VH_REGEX, vh)
if not m:
continue
pre = m.group(1)
mask = f"{pre}.mask.tif"
vv = f"{pre}.tile.vv.tif"
# Contains full path and the name for each image
l_imgs = [(mask, os.path.join(root, mask)),
(vh, os.path.join(root, vh)),
(vv, os.path.join(root, vv))]
num_imgs = int(len(files) / 3)
with gdal_open(os.path.join(root, vh)) as f:
vh_array = f.ReadAsArray()
if not valid_image(vh_array):
update_count += 1
delete_imgs(l_imgs)
continue
with gdal_open(os.path.join(root, vv)) as f:
vv_array = f.ReadAsArray()
if not valid_image(vv_array):
update_count += 1
delete_imgs(l_imgs)
continue
with gdal_open(os.path.join(root, mask)) as f:
mask_array = f.ReadAsArray()
count += 1
pyplot.subplot(1, 3, 1)
pyplot.title('mask: Water = Black Land = White')
pyplot.xlabel(f'On {count} of {num_imgs-update_count}')
pyplot.ylabel(mask)
pyplot.imshow(mask_array, cmap=pyplot.get_cmap('gist_yarg'))
pyplot.subplot(1, 3, 2)
pyplot.title('vh')
pyplot.xlabel(f'Remaining images: {num_imgs-count+1-update_count}')
flt = vh_array.flatten()
mean = flt.mean()
std = flt.std()
vh_array = vh_array.clip(0, mean + 3 * std)
pyplot.imshow(vh_array.reshape(512, 512),
cmap=pyplot.get_cmap('gist_gray'))
pyplot.subplot(1, 3, 3)
pyplot.title('vv')
flt = vh_array.flatten()
mean = flt.mean()
std = flt.std()
vv_array = vv_array.clip(0, mean + 20 * std)
pyplot.imshow(vv_array.reshape(512, 512),
cmap=pyplot.get_cmap('gist_gray'))
def close_plot(_: Any) -> None:
nonlocal done
done = True
_cbtn = close_button(close_plot)
_kpbtn = keep_button(g_path, l_imgs)
_dbtn = delete_button(l_imgs)
maximize_plot()
pyplot.show()