def removeCloud(folder, inBand):
bands = config_this.getFoldTiff(folder)
for band in bands:
if 'BQA' in band:
break
BQA = gdal.Open(band)
BQA = BQA.ReadAsArray().astype(np.float)
BQA = np.array(BQA)
bands = config_this.getFoldTiff(folder)
for band in bands:
if inBand in band:
break
print(band)
band_buffer = band
new = gdal.Open(band)
new = new.ReadAsArray().astype(np.float)
new = np.array(new)
noise = [
2976, 2980, 2984, 2988, 3008, 3012, 3016, 3020, 7072, 7076, 7080, 7084,
7104, 7108, 7112, 7116, 2800, 2804, 2808, 2812, 6896, 6900, 6904, 6908
]
for i in noise:
new[np.where(BQA == i)] = np.nan
newband = band_buffer.replace('2.TIF', '3.TIF')
print(newband)
config_this.createTiff(new, band_buffer, newband)
def LST(folder):
pathNDVI = NDVI(folder)
Tiffs = getFoldTiff(folder)
for band in Tiffs:
if 'B10' in band:
break
band_buffer = band
outband = band.replace('10.TIF', '10product2.TIF')
driver = gdal.GetDriverByName("GTiff")
band = gdal.Open(band)
band.GetRasterBand(1)
band = band.ReadAsArray().astype(np.float)
band[np.where(band < 0)] = np.NaN
config_this.createTiff(band, band_buffer, outband)
for band in Tiffs:
if 'B10product2' in band:
break
driver = gdal.GetDriverByName('GTiff')
in_file = gdal.Open(pathNDVI)
band = in_file.GetRasterBand(1)
NDVIarray = (band.ReadAsArray()).astype(np.float)
NDVIarray[np.isnan(NDVIarray)] = 0
NDVIarray[np.isinf(NDVIarray)] = 0
NDVImax, NDVImin = np.nanmax(NDVIarray), np.nanmin(NDVIarray)
pathT = folder + '/' + os.path.basename(folder) + '_BT.TIF'
pathE = folder + '/' + os.path.basename(folder) + '_BE.TIF'
pathLST = folder + '/' + os.path.basename(folder) + '_BLST.TIF'
# K1=774.89, K2=1321.08, M=3.342*(10**(-4)), A=0.1.
# lamdaB = B10*(M)+A
# T = K2/(np.log(1+K1/lamdaB))-273.15
# lDevideP=float(10.8/14388)
# e = ((x-NDVImin)/(NDVImax-NDVImin))**2
# LST = T/(1+(float(10.8/14388)*T)*np.log(e))
eqT = '1321.08/(log(1+774.89/(B10*3.342*(10**(-4))+0.1)))-273.15'
eqE = '((BNDVI-%f)/(%f-%f))**2' % (NDVImin, NDVImax, NDVImin)
eqLST = 'BT/(1+(0.000751)*BT*log(BE))'
calTiff(folder, eqT, pathT)
calTiff(folder, eqE, pathE)
calTiff(folder, eqLST, pathLST)
return (pathLST)
def calTiff(folder, expression, out_filename, NoDataValue='NaN'):
if 'NDVI' in out_filename or 'LST' in out_filename or 'BE' in out_filename or 'BT' in out_filename:
NoDataValue = '0'
#gdal accept only A -> Z as band name
BandValid = [
"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N",
"O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"
]
bandPaths = getFoldTiff(folder)
#create dictionary of {bandName:bandPath}
bandNeed = (re.findall('\w+', expression))
bandNeed2 = list()
for i in bandNeed:
if i[0] == 'B':
bandNeed2.append(i)
bandNeed = bandNeed2
band2paths = dict()
for band in bandNeed:
for bandPath in bandPaths:
if band in os.path.basename(bandPath):
band2paths.update({BandValid[bandNeed.index(band)]: bandPath})
bandPaths.remove(bandPath)
#astpye(float32) for getting negative value, and return correct number when devide
for band in set(bandNeed):
expression = expression.replace(
band, str(BandValid[bandNeed.index(band)] + '.astype(float32)'))
#assign band2paths dictionary into a gdal_calc.py string
assign = str()
for key, value in band2paths.items():
assign = assign + ' -' + key + ' ' + value
#then put all of them in a cmdLine
cmdLine = ("py -3 " + py_scripts + "/gdal_calc.py --calc=" + expression +
' --type=Float32 --NoDataValue=' + NoDataValue + ' --outfile=' +
out_filename + assign)
p = subprocess.Popen(cmdLine, shell=True)
p.wait()
print(cmdLine)
print('\n')
def run(shp, date, province):
print(shp)
driver = ogr.GetDriverByName('ESRI Shapefile')
dataSource = driver.Open(shp, 0)
layer = dataSource.GetLayer()
feature = layer[0]
geom = feature.GetGeometryRef()
A = geom.GetEnvelope()
lowLeftLat = A[2] #(y dưới trái)
lowLeftLong = A[0] #(x dưới trái)
upRightLat = A[3] #(y trên phải)
upRightLong = A[1] #(x trên phải)
print(lowLeftLat)
print(lowLeftLong)
print(upRightLat)
print(upRightLong)
#create a landsat folder
landsat_folders = ls8Down.Download_Landsat_8(str(lowLeftLat),
str(lowLeftLong),
str(upRightLat),
str(upRightLong), str(date))
# landsat_folders = [
# 'LC08_L1TP_125052_20160228_20180523_01_T1'
# ,'LC08_L1TP_125052_20140326_20170424_01_T1'
# ,'LC08_L1TP_125052_20150209_20180523_01_T1'
# ,'LC08_L1TP_125052_20170214_20170228_01_T1'
# ,'LC08_L1TP_125052_20180217_20180307_01_T1'
# ,'LC08_L1TP_125052_20190220_20190222_01_T1'
# ,'LC08_L1TP_125053_20140326_20170424_01_T1'
# ,'LC08_L1TP_125053_20150209_20180523_01_T1'
# ,'LC08_L1TP_125053_20160228_20180523_01_T1'
# ,'LC08_L1TP_125053_20180217_20180307_01_T1'
# ,'LC08_L1TP_125053_20170214_20170228_01_T1'
# ,'LC08_L1TP_125053_20190220_20190222_01_T1'
# ]
# print (landsat_folders)
#create a dictionary of all diriect folder, incase process many images once
all_sum_direct_folder = list()
#process each tar then add final to a tuple all_sum_direct_folder
for landsat_folder in landsat_folders:
direct_folder = province + 'IMAGES'
if not os.path.exists(landsat_folder):
os.mkdir(landsat_folder)
else:
pass
#extract file into a landsat folder
zipFile = landsat_folder + '.tar.gz'
tar = tarfile.open(zipFile, "r:gz")
tar.extractall(landsat_folder)
tar.close()
#create a folder has province's name, if exist then pass.
if not os.path.exists(direct_folder):
os.mkdir(direct_folder)
else:
pass
#pick a name of first TIF to get date, scene...
basename = config_this.getFoldTiff(landsat_folder)
basename = pathOfFile(basename[0])
#create a direct which will be lately moved tif into
sum_direct_folder = direct_folder + '/' + direct_folder + '_' + basename.datein
if not os.path.exists(sum_direct_folder):
os.mkdir(sum_direct_folder)
else:
pass
direct_folder = direct_folder + '/' + direct_folder + '_' + basename.datein + '_' + basename.scene
if not os.path.exists(direct_folder):
os.mkdir(direct_folder)
else:
pass
#atmospheric correction
histo.run(landsat_folder)
########
remove_cloud.removeCloud(landsat_folder, 'B2product2')
remove_cloud.removeCloud(landsat_folder, 'B3product2')
remove_cloud.removeCloud(landsat_folder, 'B4product2')
remove_cloud.removeCloud(landsat_folder, 'B5product2')
remove_cloud.removeCloud(landsat_folder, 'B10')
#clip band from landsat_folder and put result into direct_folder (province folder)
clipMosaic.clipFolder(landsat_folder,
direct_folder,
shp,
band='B2product3')
clipMosaic.clipFolder(landsat_folder,
direct_folder,
shp,
band='B3product3')
clipMosaic.clipFolder(landsat_folder,
direct_folder,
shp,
band='B4product3')
clipMosaic.clipFolder(landsat_folder,
direct_folder,
shp,
band='B5product3')
clipMosaic.clipFolder(landsat_folder, direct_folder, shp, band='B10')
clipMosaic.clipFolder(landsat_folder, direct_folder, shp, band='BQA')
#remove zipfile and landsat_folder
# os.remove(zipFile)
shutil.rmtree(landsat_folder)
#cal NDVI and LST then write a record to postgreSQL
cal.NDVI(direct_folder)
cal.LST(direct_folder)
config_this.bring(direct_folder, sum_direct_folder)
shutil.rmtree(direct_folder)
#add sumdirect folder to all_direct_folder incase many process once
all_sum_direct_folder.append(sum_direct_folder)
all_sum_direct_folder = tuple(all_sum_direct_folder)
#mosaic for each folder
for sum_direct_folder in all_sum_direct_folder:
# clipMosaic.MosaicFolder(sum_direct_folder,'B2ref2')
# clipMosaic.MosaicFolder(sum_direct_folder,'B3ref2')
# clipMosaic.MosaicFolder(sum_direct_folder,'B4ref2') #mosaic after have folder for true color
#for process
clipMosaic.MosaicFolder(sum_direct_folder, 'BNDVI')
clipMosaic.MosaicFolder(sum_direct_folder, 'BLST')
Tifs = config_this.getFoldTiff(sum_direct_folder)
for Tif in Tifs:
if 'BNDVI' in Tif:
pathNDVI = Tif
elif 'BLST' in Tif:
pathLST = Tif
else:
os.remove(Tif)
fieldsValues = {
'location':
pathNDVI,
'the_geom':
connectPostgres.getGeom(lowLeftLong, lowLeftLat, upRightLong,
upRightLat, 4326)
}
connectPostgres.insertSQL('processed_images', **fieldsValues)
fieldsValues = {
'location':
pathLST,
'the_geom':
connectPostgres.getGeom(lowLeftLong, lowLeftLat, upRightLong,
upRightLat, 4326)
}
connectPostgres.insertSQL('processed_images', **fieldsValues)
def run(shp, date, province):
print(shp)
driver = ogr.GetDriverByName('ESRI Shapefile')
dataSource = driver.Open(shp, 0)
layer = dataSource.GetLayer()
feature = layer[0]
geom = feature.GetGeometryRef()
A = geom.GetEnvelope()
lowLeftLat = A[2] #(y dưới trái)
lowLeftLong = A[0] #(x dưới trái)
upRightLat = A[3] #(y trên phải)
upRightLong = A[1] #(x trên phải)
print(lowLeftLat)
print(lowLeftLong)
print(upRightLat)
print(upRightLong)
#create a landsat folder
landsat_folders = ls8Down.Download_Landsat_8(str(lowLeftLat),
str(lowLeftLong),
str(upRightLat),
str(upRightLong), str(date))
# print (landsat_folders)
# create a dictionary of all diriect folder, incase process many images once
# all_sum_direct_folder = list()
# process each tar then add final to a tuple all_sum_direct_folder
for landsat_folder in landsat_folders:
direct_folder = province + 'IMAGES'
NDVI_folder = direct_folder + '/' + direct_folder + '_NDVI'
LST_folder = direct_folder + '/' + direct_folder + '_LST'
if not os.path.exists(landsat_folder):
os.mkdir(landsat_folder)
else:
pass
#extract file into a landsat folder
zipFile = landsat_folder + '.tar.gz'
tar = tarfile.open(zipFile, "r:gz")
tar.extractall(landsat_folder)
tar.close()
#create a folder has province's name, if exist then pass.
if not os.path.exists(direct_folder):
os.mkdir(direct_folder)
else:
pass
#pick a name of first TIF to get date, scene...
basename = config_this.getFoldTiff(landsat_folder)
basename = pathOfFile(basename[0])
#create a direct which will be lately moved tif into
sum_direct_folder = direct_folder + '/' + direct_folder + '_' + basename.datein
if not os.path.exists(sum_direct_folder):
os.mkdir(sum_direct_folder)
else:
pass
direct_folder = direct_folder + '/' + direct_folder + '_' + basename.datein + '_' + basename.scene
if not os.path.exists(direct_folder):
os.mkdir(direct_folder)
else:
pass
#atmospheric correction
histo.run(landsat_folder)
########
remove_cloud.removeCloud(landsat_folder, 'B2product2')
remove_cloud.removeCloud(landsat_folder, 'B3product2')
remove_cloud.removeCloud(landsat_folder, 'B4product2')
remove_cloud.removeCloud(landsat_folder, 'B5product2')
remove_cloud.removeCloud(landsat_folder, 'B10')
#clip band from landsat_folder and put result into direct_folder (province folder)
clipMosaic.clipFolder(landsat_folder,
direct_folder,
shp,
band='B2product3')
clipMosaic.clipFolder(landsat_folder,
direct_folder,
shp,
band='B3product3')
clipMosaic.clipFolder(landsat_folder,
direct_folder,
shp,
band='B4product3')
clipMosaic.clipFolder(landsat_folder,
direct_folder,
shp,
band='B5product3')
clipMosaic.clipFolder(landsat_folder, direct_folder, shp, band='B10')
clipMosaic.clipFolder(landsat_folder, direct_folder, shp, band='BQA')
#remove zipfile and landsat_folder
# os.remove(zipFile)
shutil.rmtree(landsat_folder)
#cal NDVI and LST then write a record to postgreSQL
cal.NDVI(direct_folder)
cal.LST(direct_folder)
config_this.bring(direct_folder, sum_direct_folder)
shutil.rmtree(direct_folder)
if not os.path.exists(NDVI_folder):
os.mkdir(NDVI_folder)
else:
pass
if not os.path.exists(LST_folder):
os.mkdir(LST_folder)
else:
pass
clipMosaic.MosaicFolder(sum_direct_folder, 'BNDVI')
clipMosaic.MosaicFolder(sum_direct_folder, 'BLST')
Tifs = config_this.getFoldTiff(sum_direct_folder)
for Tif in Tifs:
if 'BNDVI' in Tif:
new_tif = NDVI_folder + '/' + Tif.split('/')[-1]
pathNDVI = new_tif
shutil.move(Tif, new_tif)
elif 'BLST' in Tif:
new_tif = LST_folder + '/' + Tif.split('/')[-1]
pathLST = new_tif
shutil.move(Tif, new_tif)
else:
os.remove(Tif)
os.remove(sum_direct_folder)
print('Toi_day_chua_xoa')
fieldsValues = {
'location':
pathNDVI,
'the_geom':
connectPostgres.getGeom(lowLeftLong, lowLeftLat, upRightLong,
upRightLat, 4326)
}
connectPostgres.insertSQL('processed_images', **fieldsValues)
fieldsValues = {
'location':
pathLST,
'the_geom':
connectPostgres.getGeom(lowLeftLong, lowLeftLat, upRightLong,
upRightLat, 4326)
}
connectPostgres.insertSQL('processed_images', **fieldsValues)