def rastertocsv(directory,
filename='PyRSGIS_rasterToCSV.csv',
negative=True,
badrows=True):
os.chdir(directory)
data = []
names = []
for file in glob.glob("*.tif"):
ds, band = read(file, band=1)
nBands = ds.RasterCount
for n in range(1, nBands + 1):
names.append(file[:-4] + "@" + str(n))
ds, band = read(file, band=n)
if negative == False:
band[band < 0] = 0
band = onedimension(band)
data.append(band)
dataArray = np.array(data)
dataArray = np.transpose(dataArray)
if badrows == False:
dataArray[~np.isfinite(
dataArray)] = 0 #Replacing all the infinite and nan values to zero
dataArray = dataArray[~(dataArray == 0).all(1)]
with open(filename, 'w', newline='') as csvfile:
writer = csv.writer(csvfile, delimiter=',')
writer.writerow(names)
for row in range(0, dataArray.shape[0]):
rowData = dataArray[row]
writer.writerow(rowData)
def forclassification():
ds, slp = raster.read("slope-prediction.tif")
ds, img = raster.read("prediction.tif")
slp = np.reshape(slp, (slp.shape[0] * slp.shape[1], 1)).astype(int)
for i in range(img.shape[0]):
img1 = np.zeros((slp.shape[0], img.shape[0]), dtype=int)
img1[:, i] = (np.reshape(img[i, :, :], (slp.shape[0]))).astype(int)
stack = np.concatenate((img1, slp), axis=1)
return (stack)
def preprocessing():
#reading data, ds includes satellite iamges data structure that is necessary for exporting the ruslt in TIF format
ds, inv = raster.read(inventory)
ds, slp = raster.read(slope)
ds, s2A = raster.read(img)
#reshaping data
inv = np.reshape(inv, (inv.shape[0] * inv.shape[1], 1)).astype(int)
slp = (np.reshape(slp, inv.shape)).astype(int)
#reshape n dimensional satellite data to 2d array. each column represents an image band or feature
for i in range(s2A.shape[0]):
s2 = np.zeros((s2A.shape[1] * s2A.shape[2], s2A.shape[0]))
s2[:, i] = (np.reshape(s2A[i, :, :],
(s2A.shape[1] * s2A.shape[2]))).astype(int)
stack = np.concatenate((s2, slp), axis=1)
print(f'the shape of labeled data: {inv.shape} ')
#print (f'the shape of image features: {s2A.shape} ')
#print (f'the shape of slope layer: {slp.shape} ')
print(
f'image features and slope layer are stacked together with the shape of: {stack.shape}'
)
return (stack, inv)
import numpy as np
from pyrsgis import raster
#####################################################################
##### PART - A: CREATING AND STORING IMAGE CHIPS AS NUMPY ARRAYS ####
#####################################################################
# Change the working directory
output_directory = r"E:\TDS_CNN"
os.chdir(output_directory)
# Read the feature and label rasters
feature_raster_file = r"Jiaxing_2015_Landsat.tif"
label_raster_file = r"Jiaxing_2015_builtup.tif"
ds, feature_raster = raster.read(feature_raster_file)
ds, label_raster = raster.read(label_raster_file)
if (feature_raster.shape[-1] != label_raster.shape[-1]) or\
(feature_raster.shape[-2] != label_raster.shape[-2]):
print('Shape of the input rasters do not match. Ending program.')
else:
print("Rasters' shape matched.")
# Generate image chips in the back-end
def CNNdataGenerator(mxBands, labelBand, kSize):
mxBands = mxBands / 255.0
nBands, rows, cols = mxBands.shape
margin = math.floor(kSize / 2)
mxBands = np.pad(mxBands, margin,
import pandas as pd
import matplotlib.pyplot as plt
import scipy
os.chdir("C:\\Users\Hugo\Desktop\Data for good\Images\\32 bits - 1ere sessions")
# premier image : seredou 2017
MSI = 'seredou_20170205_MSI.tif'
all_bands='seredou_20170205_allbands.tif'
truth_1='seredou_20170205_truth.tif'
# Read the rasters as array
ds1, feature_MSI = raster.read(MSI, bands='all')
ds2, feature_all_bands = raster.read(all_bands, bands='all')
ds3, truth_1= raster.read(truth_1, bands='all')
# sur all bands
#feature_all_bands=np.delete(feature_all_bands,np.s_[8951:9120],axis=2)
# sur truth
#truth=np.delete(truth,6141,axis=0)
# sur MSI
#feature_MSI=np.delete(feature_MSI,8951,axis=1)
# test min max des 3 images+
print("Max pour truth:", truth_1.max() )
import numpy as np
from pyrsgis import raster
#####################################################################
##### PART - A: READING AND STORING IMAGE CHIPS AS NUMPY ARRAYS #####
#####################################################################
# Change the working directory
imageDirectory = r"E:\CNN_Builtup\TDS_CNN\11by11\ImageChips"
os.chdir(imageDirectory)
# Get the number of files in the directory
nSamples = len(glob.glob('*.tif'))
# Get basic information about the image chips
ds, tempArr = raster.read(os.listdir(imageDirectory)[0])
nBands, rows, cols = ds.RasterCount, ds.RasterXSize, ds.RasterYSize
# Create empty arrays to store data later
features = np.empty((nSamples, nBands, rows, cols))
labels = np.empty((nSamples, ))
# Loop through the files, read and stack
for n, file in enumerate(glob.glob('*.tif')):
if n == nSamples:
break
if n % 5000 == 0:
print('Sample read: %d of %d' % (n, nSamples))
ds, tempArr = raster.read(file)
# Get filename without extension, split by underscore and get the label
tempLabel = os.path.splitext(file)[0].split('_')[-1]
import numpy as np
from pyrsgis import raster
from scipy.stats.stats import pearsonr
RasterFile1 = 'e:/ndvi.tif'
RasterFile2 = 'e:/prec.tif'
ds1, tempArr1 = raster.read(RasterFile1)
ds2, tempArr2 = raster.read(RasterFile2)
nBands1, rows1, cols1 = ds1.RasterCount, ds1.RasterXSize, ds1.RasterYSize
features1 = np.empty((nBands1, cols1, rows1))
features2 = np.empty((nBands1, cols1, rows1))
features1[:, :, :] = tempArr1
features2[:, :, :] = tempArr2
features_result1 = np.empty((cols1, rows1))
features_result2 = np.empty((cols1, rows1))
for i in range(0, cols1):
for j in range(0, rows1):
list1 = []
list2 = []
for k in range(0, nBands1):
list1.append(features1[k, i, j])
list2.append(features2[k, i, j])
r, p_value = pearsonr(list1, list2)
features_result1[i, j] = r
features_result2[i, j] = p_value
outFile1 = 'd:/aa.tif'
outFile2 = 'd:/bb.tif'
raster.export(features_result1,
ds1,
filename=outFile1,
#convert all tif to jpgs
#install libraries first
#!pip install pyrsgis #(for collab -> uncomment this line)
import numpy as np
import glob
import cv2
from pyrsgis import raster
for i in range(1): #put number of images here
images = glob.glob(
'*.tif'
) #path of images #if possible put in same folder and execute. might cause some error.
for image in images:
temp = image.split(".")[0]
print(temp)
new_name = temp + ".jpg"
ds2, chip = raster.read(image, bands=[1, 2, 3])
t1 = (chip[0]).astype(np.uint16)
t2 = (chip[1]).astype(np.uint16)
t3 = (chip[2]).astype(np.uint16)
tdstck = np.dstack((t3, t2, t1))
cv2.imwrite(new_name, tdstck)
"""
######################################################################
##### PART - C: LOADING THE SAVED MODEL AND PREDICTING NEW IMAGE #####
######################################################################
# Change the working directory
os.chdir(r'E:\CNN_Builtup\TDS_CNN')
# Load the saved model
model = tf.keras.models.load_model(
'trained_models/200409_CNN_Builtup_11by11_PScore0.928_RScore0.936_FScore0.932.h5'
)
# Load a new multispectral image
ds, featuresHyderabad = raster.read('l5_Hyderabad2011_raw.tif')
outFile = 'l5_Hyderabad_2011_Builtup_CNN_predicted_11by11.tif'
# Generate image chips in the back-end
def CNNdataGenerator(mxBands, kSize):
mxBands = mxBands / 255.0
nBands, rows, cols = mxBands.shape
margin = math.floor(kSize / 2)
mxBands = np.pad(mxBands, margin, mode='constant')[margin:-margin, :, :]
features = np.empty((rows * cols, kSize, kSize, nBands))
n = 0
for row in range(margin, rows + margin):
for col in range(margin, cols + margin):
from tensorflow import keras
from pyrsgis import raster
from pyrsgis.convert import changeDimension
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, precision_score, recall_score
# Change the directory
os.chdir("E:\\BuiltUpPrediction")
# Assign file names
mxBangalore = 'l5_Bangalore2011_raw.tif'
builtupBangalore = 'l5_Bangalore2011_builtup.tif'
mxHyderabad = 'l5_Hyderabad2011_raw.tif'
# Read the rasters as array
ds1, featuresBangalore = raster.read(mxBangalore, bands='all')
ds2, labelBangalore = raster.read(builtupBangalore, bands=1)
ds3, featuresHyderabad = raster.read(mxHyderabad, bands='all')
# Print the size of the arrays
print("Bangalore Multispectral image shape: ", featuresBangalore.shape)
print("Bangalore Binary built-up image shape: ", labelBangalore.shape)
print("Hyderabad Multispectral image shape: ", featuresHyderabad.shape)
# Clean the labelled data to replace NoData values by zero
labelBangalore = (labelBangalore == 1).astype(int)
# Reshape the array to single dimensional array
featuresBangalore = changeDimension(featuresBangalore)
labelBangalore = changeDimension(labelBangalore)
featuresHyderabad = changeDimension(featuresHyderabad)
forest.fit(xtrain, ytrain)
fpred = forest.predict(xtest)
#model performance evaluation
print("random forest: ")
print(classification_report(ytest, fpred))
return (forest)
training()
##################################################################################################
#classifiying the image
ds, img2 = raster.read("prediction.tif")
def prediction():
#recalling data
data = forclassification()
#recalling trained model
model = training()
a = model.predict(data)
print(a.shape)
#reshaping the prediction or classification result to original image size
clmap = np.reshape(a, (img2.shape[1], img2.shape[2]))
#converting array into TIF map
output = "landslide_detection_map.tif"
raster.export(clmap, ds, filename=output, dtype="int")
return (clmap, clmap.shape)
# define the file names
feature_file = r"E:\CNN_Builtup\l5_Bangalore2011_raw.tif"
label_file = r"E:\CNN_Builtup\l5_Bangalore2011_builtup.tif"
# create feature chips using pyrsgis
features = imageChipsFromFile(feature_file, x_size=7, y_size=7)
""" Update: 29 May 2021
Since I added this code chunk later, I wanted to make least
possible changes in the remaining sections. The below line changes
the index of the channels. This will be undone at a later stage.
"""
features = np.rollaxis(features, 3, 1)
# read the label file and reshape it
ds, labels = raster.read(label_file)
labels = labels.flatten()
# check for irrelevant values (we are interested in 1s and non-1s)
labels = (labels == 1).astype(int)
# print basic details
print('Input features shape:', features.shape)
print('\nInput labels shape:', labels.shape)
print('Values in input features, min: %d & max: %d' %
(features.min(), features.max()))
print('Values in input labels, min: %d & max: %d' %
(labels.min(), labels.max()))
# Save the arrays as .npy files
np.save('CNN_7by7_features.npy', features)