def create_anomaly_raster(name, input_dir, output_dir):
epoch_start = 2002
this_year = int(name[:4])
if (os.path.exists(os.path.join(output_dir, name))):
print("Anomaly file %s already exists" % name)
return
image_stack = []
for year in range(this_year - 10, this_year):
if (os.path.exists(os.path.join(input_dir, str(year) + name[4:]))):
image_stack.append(os.path.join(input_dir, str(year) + name[4:]))
if (len(image_stack) == 10):
break
if (len(image_stack) < 10):
return
print("Computing historical anomaly for %s dataset" % name)
import gdal
import osr
import ogr
import numpy as np
from functools import reduce
rasters = [gdal.Open(file_path) for file_path in image_stack]
current_raster = gdal.Open(os.path.join(input_dir, name))
ysize = rasters[0].RasterYSize
xsize = rasters[0].RasterXSize
GeoT = rasters[0].GetGeoTransform()
Projection = rasters[0].GetProjection()
NDV = rasters[0].GetRasterBand(1).GetNoDataValue()
bands = [raster.GetRasterBand(1) for raster in rasters]
array_rasters = [band.ReadAsArray() for band in bands]
array_sum = np.sum(array_rasters, axis=0)
array_mean = array_sum / 10
array_std = np.zeros((ysize, xsize))
for raster in array_rasters:
array_std += (raster - array_mean)**2
del (raster)
array_std = np.sqrt(array_std / 10)
del (rasters)
del (bands)
del (array_rasters)
gc.collect()
current_value = current_raster.GetRasterBand(1).ReadAsArray()
array_out = (array_mean - current_value) / array_std
# array_std = ((array_sum ** 2)/10 - (array_mean)**2)
array_out[current_value == NDV] = NDV
new_file_name = name
driver = gdal.GetDriverByName('GTiff')
DataSet = driver.Create(os.path.join(output_dir, new_file_name), xsize,
ysize, 1, gdal.GDT_Byte)
DataSet.SetGeoTransform(GeoT)
srs = osr.SpatialReference()
srs.ImportFromEPSG(4326)
DataSet.SetProjection(srs.ExportToWkt())
DataSet.GetRasterBand(1).WriteArray(array_out)
DataSet.GetRasterBand(1).SetNoDataValue(NDV)
DataSet.FlushCache()
DataSet = None
print("created anomaly file %s" % new_file_name)
return
def array_to_geotiff(fname,
data,
geo_transform,
projection,
nodata_val=0,
dtype=gdal.GDT_Float32):
"""
Create a single band GeoTIFF file with data from an array.
Because this works with simple arrays rather than xarray datasets
from DEA, it requires geotransform info ("(upleft_x, x_size,
x_rotation, upleft_y, y_rotation, y_size)") and projection data
(in "WKT" format) for the output raster. These are typically
obtained from an existing raster using the following GDAL calls:
import gdal
gdal_dataset = gdal.Open(raster_path)
geotrans = gdal_dataset.GetGeoTransform()
prj = gdal_dataset.GetProjection()
...or alternatively, directly from an xarray dataset:
geotrans = xarraydataset.geobox.transform.to_gdal()
prj = xarraydataset.geobox.crs.wkt
Parameters
----------
fname : str
Output geotiff file path including extension
data : numpy array
Input array to export as a geotiff
geo_transform : tuple
Geotransform for output raster; e.g. "(upleft_x, x_size,
x_rotation, upleft_y, y_rotation, y_size)"
projection : str
Projection for output raster (in "WKT" format)
nodata_val : int, optional
Value to convert to nodata in the output raster; default 0
dtype : gdal dtype object, optional
Optionally set the dtype of the output raster; can be
useful when exporting an array of float or integer values.
Defaults to gdal.GDT_Float32
"""
# Set up driver
driver = gdal.GetDriverByName('GTiff')
# Create raster of given size and projection
rows, cols = data.shape
dataset = driver.Create(fname, cols, rows, 1, dtype)
dataset.SetGeoTransform(geo_transform)
dataset.SetProjection(projection)
# Write data to array and set nodata values
band = dataset.GetRasterBand(1)
band.WriteArray(data)
band.SetNoDataValue(nodata_val)
# Close file
dataset = None
def gbbvi(OS_landuse_path, LWF_path, TCD_20_path, LCM_path, ceh_con_path,
ukveg_out, ukveg_clip, water_buff_path, folder):
print("importing data and converting to arrays")
# convert all rasters to numpy arrays with Gdal
if os.path.isfile(OS_landuse_path):
OS_landuse = gdal.Open(OS_landuse_path)
OS_landuse_Band = OS_landuse.GetRasterBand(
1) # This is here to allow for the export at the end
OS_landuse_Arr = BandReadAsArray(OS_landuse_Band)
OS_landuse_Arr[OS_landuse_Arr > 5] = 999
else:
print("No OS Vector data for Grid {0} - Skip grid".format(folder))
return
if os.path.isfile(LWF_path):
LWF = gdal.Open(LWF_path)
LWF_Arr = np.array(LWF.GetRasterBand(1).ReadAsArray())
LWF_Arr[LWF_Arr > 5] = 999
else:
print("No LWFs for Grid {0}".format(folder))
if os.path.isfile(TCD_20_path):
TCD_20 = gdal.Open(TCD_20_path)
TCD_20_Arr = np.array(TCD_20.GetRasterBand(1).ReadAsArray())
TCD_20_Arr[TCD_20_Arr > 100] = 999
else:
print("No Tree Cover Density data for Grid {0} - Skip grid".format(
folder))
return
if os.path.isfile(LCM_path):
LCM = gdal.Open(LCM_path)
LCM_Arr = np.array(LCM.GetRasterBand(1).ReadAsArray())
LCM_Arr[LCM_Arr > 5] = 999
else:
print("No Land Cover data for Grid {0} - Skip grid".format(folder))
return
if os.path.isfile(ceh_con_path):
ceh_con = gdal.Open(ceh_con_path)
ceh_con_Arr = np.array(ceh_con.GetRasterBand(1).ReadAsArray())
ceh_con_Arr[ceh_con_Arr != 3] = 999
else:
print("No Conifers for Grid {0}".format(folder))
if os.path.isfile(water_buff_path):
wat_buf = gdal.Open(water_buff_path)
wat_buf_Arr = np.array(wat_buf.GetRasterBand(1).ReadAsArray())
wat_buf_Arr[wat_buf_Arr != 1] = 0
else:
print("No Water for Grid {0}".format(folder))
return
print("running Tree Cover Density recalculation")
TCD_20_Arr[np.isnan(TCD_20_Arr)] = 0
TCD_20_Arr[TCD_20_Arr == 0] = 0
TCD_20_Arr[TCD_20_Arr > 100] = 0
TCD_20_Arr[(TCD_20_Arr > 0) & (TCD_20_Arr < 3)] = 1
TCD_20_Arr[(TCD_20_Arr >= 3) & (TCD_20_Arr < 10)] = 2
TCD_20_Arr[(TCD_20_Arr >= 10) & (TCD_20_Arr < 50)] = 3
TCD_20_Arr[(TCD_20_Arr >= 50) & (
TCD_20_Arr <= 100
)] = 4 # Consider upping this to 5 now that we have the conif layer
# tcd_a = TCD_20_Arr
print("running primary BVI")
vegval = np.zeros_like(
OS_landuse_Arr) # create a new np array of equal size to rasters
vegval[TCD_20_Arr == 0] = LCM_Arr[TCD_20_Arr == 0]
vegval[(TCD_20_Arr > 0)
& (TCD_20_Arr <= 5)] = TCD_20_Arr[(TCD_20_Arr > 0)
& (TCD_20_Arr <= 5)]
if os.path.isfile(LWF_path):
vegval[(LWF_Arr > 0) & (LWF_Arr <= 5)] = LWF_Arr[(LWF_Arr > 0)
& (LWF_Arr <= 5)]
vegval[OS_landuse_Arr <= 5] = OS_landuse_Arr[OS_landuse_Arr <= 5]
print("running secondary BVI")
vegval[(vegval < LCM_Arr)] = LCM_Arr[(vegval < LCM_Arr)]
vegval[LCM_Arr > 5] = 0
if os.path.isfile(LWF_path):
vegval[vegval < LWF_Arr] = LWF_Arr[vegval < LWF_Arr]
if os.path.isfile(ceh_con_path):
vegval[(vegval <= TCD_20_Arr) & (ceh_con_Arr != 3)] = TCD_20_Arr[(
vegval <= TCD_20_Arr
) & (
ceh_con_Arr != 3
)] # be aware that if we don't make cehcon 3 in prior script this will fail
vegval[OS_landuse_Arr < 1] = OS_landuse_Arr[OS_landuse_Arr < 1]
print("exporting BVI Raster to " + str(ukveg_out))
driver = gdal.GetDriverByName("GTiff")
gb_bvi = driver.Create(ukveg_out, OS_landuse.RasterXSize,
OS_landuse.RasterYSize, 1, OS_landuse_Band.DataType)
CopyDatasetInfo(OS_landuse, gb_bvi)
bandOut = gb_bvi.GetRasterBand(1)
BandWriteArray(bandOut, vegval)
print("BVI Raster for OS GRID {0} exported".format(folder))
# Here we need to clip the BVI to the buffered area
print("cropping area outside buffer for OS {0}".format(folder))
vegval[wat_buf_Arr != 1] = 0
print("exporting BHI Raster to " + str(ukveg_clip))
driverc = gdal.GetDriverByName("GTiff")
gb_bvic = driverc.Create(ukveg_clip, OS_landuse.RasterXSize,
OS_landuse.RasterYSize, 1,
OS_landuse_Band.DataType)
CopyDatasetInfo(OS_landuse, gb_bvic)
bandOutc = gb_bvic.GetRasterBand(1)
BandWriteArray(bandOutc, vegval)
print("BHI Raster for OS GRID {0} exported".format(folder))
# Close the data sets
OS_landuse_Band = None
OS_landuse = None
LWF = None
TCD_20 = None
LCM = None
ceh_con = None
bandOut = None
gb_bvi = None
bandOutc = None
gb_bvic = None
wat_buf = None
print(datetime.now() - startTime)
print("output created in: " + str(ukveg_out))
def zonal_stats(feat, input_zone_polygon, input_value_raster):
# Open data
raster = gdal.Open(input_value_raster)
shp = ogr.Open(input_zone_polygon)
lyr = shp.GetLayer()
# Get raster georeference info
transform = raster.GetGeoTransform()
xOrigin = transform[0]
yOrigin = transform[3]
pixelWidth = transform[1]
pixelHeight = transform[5]
# Get extent of feat
geom = feat.GetGeometryRef()
if (geom.GetGeometryName() == 'MULTIPOLYGON'):
count = 0
pointsX = []
pointsY = []
for polygon in geom:
geomInner = geom.GetGeometryRef(count)
ring = geomInner.GetGeometryRef(0)
numpoints = ring.GetPointCount()
for p in range(numpoints):
lon, lat, z = ring.GetPoint(p)
pointsX.append(lon)
pointsY.append(lat)
count += 1
elif (geom.GetGeometryName() == 'POLYGON'):
ring = geom.GetGeometryRef(0)
numpoints = ring.GetPointCount()
pointsX = []
pointsY = []
for p in range(numpoints):
lon, lat, z = ring.GetPoint(p)
pointsX.append(lon)
pointsY.append(lat)
else:
sys.exit()
xmin = min(pointsX)
xmax = max(pointsX)
ymin = min(pointsY)
ymax = max(pointsY)
# Specify offset and rows and columns to read
xoff = int((xmin - xOrigin) / pixelWidth)
yoff = int((yOrigin - ymax) / pixelWidth)
xcount = int((xmax - xmin) / pixelWidth) + 1
ycount = int((ymax - ymin) / pixelWidth) + 1
# Create memory target raster
target_ds = gdal.GetDriverByName('MEM').Create('id', xcount, ycount,
gdal.GDT_Byte)
target_ds.SetGeoTransform((
xmin,
pixelWidth,
0,
ymax,
0,
pixelHeight,
))
# Create for target raster the same projection as for the value raster
raster_srs = osr.SpatialReference()
raster_srs.ImportFromWkt(raster.GetProjectionRef())
target_ds.SetProjection(raster_srs.ExportToWkt())
# Rasterize zone polygon to raster
gdal.RasterizeLayer(target_ds, [1], lyr, burn_values=[1])
# Read raster as arrays
banddataraster = raster.GetRasterBand(1)
dataraster = banddataraster.ReadAsArray(xoff, yoff, xcount,
ycount).astype(numpy.float)
bandmask = target_ds.GetRasterBand(1)
datamask = bandmask.ReadAsArray(0, 0, xcount, ycount).astype(numpy.float)
# Mask zone of raster
zoneraster = numpy.ma.masked_array(dataraster, numpy.logical_not(datamask))
# Calculate statistics of zonal raster
return numpy.mean(zoneraster)
def main(vswirfile, dicamfile, maskfile, hyperfile):
########################################################
# Open data
vswir = gdal.Open(vswirfile)
dicam = gdal.Open(dicamfile)
mask = gdal.Open(maskfile)
geotrans = dicam.GetGeoTransform()
vswirmeta = vswir.GetMetadata('ENVI')
xoff = 0
yoff = 0
xcountvswir = vswir.RasterXSize
ycountvswir = vswir.RasterYSize
xcountmask = mask.RasterXSize
ycountmask = mask.RasterYSize
xcountdicam = dicam.RasterXSize
ycountdicam = dicam.RasterYSize
# Read raster as arrays
vswirdata = vswir.ReadAsArray(xoff, yoff, xcountvswir,
ycountvswir).astype(np.float)
dicamdata = dicam.ReadAsArray(xoff, yoff, xcountdicam,
ycountdicam).astype(np.float)
maskdata = mask.ReadAsArray(xoff, yoff, xcountmask,
ycountmask).astype(np.float)
# average dicam data for the RGB bands down to vswir spatial resolution
dicamcoarse = np.zeros((3, ycountvswir, xcountvswir), dtype=np.float)
## red = 45
## green = 27
## blue = 9
ratio = np.double(10)
for j in range(ycountvswir):
for k in range(xcountvswir):
## first, find where the shaded data are to avoid
tempdatared = dicamdata[0,np.floor(j*ratio).astype(int):np.floor(j*ratio+ratio).astype(int),\
np.floor(k*ratio).astype(int):np.floor(k*ratio+ratio).astype(int)]
tempdatagreen = dicamdata[1,np.floor(j*ratio).astype(int):np.floor(j*ratio+ratio).astype(int),\
np.floor(k*ratio).astype(int):np.floor(k*ratio+ratio).astype(int)]
tempdatablue = dicamdata[2,np.floor(j*ratio).astype(int):np.floor(j*ratio+ratio).astype(int),\
np.floor(k*ratio).astype(int):np.floor(k*ratio+ratio).astype(int)]
dicamcoarse[0, j, k] = np.mean(tempdatared)
dicamcoarse[1, j, k] = np.mean(tempdatagreen)
dicamcoarse[2, j, k] = np.mean(tempdatablue)
del tempdatared, tempdatagreen, tempdatablue
good = np.equal(maskdata, 1)
numgood = good.sum()
print "Number of Good pixels: %d" % (numgood)
roworig, colorig = np.indices(good.shape)
row = roworig[good]
col = colorig[good]
vswirdata = vswirdata[:, row, col]
dicamcoarse = dicamcoarse[:, row, col]
print "Reshaped arrays"
print "DiCAM: %d %d" % dicamcoarse.shape
print "VSWIR: %d %d" % vswirdata.shape
GGt = np.linalg.inv(np.matmul(dicamcoarse, dicamcoarse.T))
B = np.matmul(np.matmul(vswirdata, dicamcoarse.T), GGt)
print "Did matrix multiplication"
pdb.set_trace()
## create a "checkerboard" mask so that the high resolution data can be ordered in rows
## to match the low re solution data.
checkers = np.arange(1, (xcountvswir * ycountvswir) + 1).reshape(
ycountvswir, xcountvswir)
checkers100 = np.kron(checkers, np.ones((10, 10))).astype(np.int64)
goodbig = np.kron(good, np.ones((10, 10))).astype(bool)
checkers100good = checkers100[goodbig]
sortindex = np.argsort(checkers100good)
rowbig, colbig = np.indices(goodbig.shape)
pdb.set_trace()
rowbig = rowbig[goodbig].flatten()
colbig = colbig[goodbig].flatten()
rowbig = rowbig[sortindex]
colbig = colbig[sortindex]
print "Created and sorted Big indices"
pdb.set_trace()
## create a numpy memory mapped file, instead of holding array into
## hypercam = np.zeros((52,ycountdicam,xcountdicam), dtype=np.float)
randnum = ("%06d") % np.random.randint(0, high=100000)
tempfile = path.join(mkdtemp(), randnum + '.dat')
hypercam = np.memmap(tempfile,
dtype='float32',
mode='w+',
shape=(52, ycountdicam, xcountdicam))
hypercam[:, rowbig, colbig] = np.matmul(B, dicamdata[:, rowbig, colbig])
print "Filled Hypercam array"
pdb.set_trace()
# Create for target raster the same projection as for the value raster
driver = gdal.GetDriverByName('ENVI')
target_ds = driver.Create(hyperfile, xcountdicam, ycountdicam,
vswir.RasterCount, gdal.GDT_Float32)
raster_srs = osr.SpatialReference()
raster_srs.ImportFromWkt(vswir.GetProjectionRef())
target_ds.SetProjection(raster_srs.ExportToWkt())
target_ds.SetGeoTransform(geotrans)
target_ds.SetMetadataItem('wavelength_units',
vswirmeta['wavelength_units'], 'ENVI')
target_ds.SetMetadataItem('wavelength', vswirmeta['wavelength'], 'ENVI')
for c in range(vswir.RasterCount):
target_ds.GetRasterBand(c + 1).WriteArray(hypercam[c, :, :])
del target_ds
del vswir, dicam, hypercam
## remove temporary directory and file
try:
shutil.rmtree(os.path.dirname(tempfile))
except:
pass
def viability_index(path, ind, year):
gdalwarp = "C:\Python276\Lib\site-packages\osgeo\gdalwarp.exe"
season = str(year) + "-" + str(year + 1)
newdir = path + "sta\\" + season
if os.path.isdir(newdir) == False:
os.mkdir(newdir)
#snow_data
snow_file = path + "snow_" + season + ".tif"
snowdays_rast = gdal.Open(snow_file)
snowdays_trans = snowdays_rast.GetGeoTransform()
snowdays = snowdays_rast.GetRasterBand(1)
#prepare sql
myconn = psycopg2.connect("host=" + conn_param.host + " dbname=" +
conn_param.dbname + " user="******" password="******"""
create table if not exists stations.viability_index(
ind varchar(4), season integer, index float8)
""")
myconn.commit
query = "delete from stations.viability_index where season= %s and ind= %s"
viability.execute(query, (
year,
ind,
))
myconn.commit
#source data and raster properties
src_ind = path + "sta\\" + ind + ".tif"
src_rast = gdal.Open(src_ind)
src_trans = src_rast.GetGeoTransform()
src_proj = src_rast.GetProjection()
src_band = src_rast.GetRasterBand(1)
mp_data = src_band.ReadAsArray()
xsize = src_band.XSize
ysize = src_band.YSize
#Create raster ind_snowdays_season
cur = myconn.cursor()
query = """
with a as (
select ind, st_envelope(st_buffer(the_geom,25)) geom
from stations.geo_dsa
where ind = %s
order by 1
)
select st_xmin(geom), st_ymin(geom), st_xmax(geom), st_ymax(geom), ind from a
"""
cur.execute(query, (ind, ))
sta = cur.fetchone()
x0 = src_trans[0] + (round(
(sta[0] - src_trans[0]) / src_trans[1]) * src_trans[1])
x1 = (src_trans[0] + src_rast.RasterXSize * src_trans[1]) - (round(
((src_trans[0] + src_rast.RasterXSize * src_trans[1]) - sta[2]) /
src_trans[1]) * src_trans[1])
y1 = src_trans[3] - (round(
(src_trans[3] - sta[3]) / src_trans[5]) * src_trans[5])
y0 = (src_trans[3] - src_rast.RasterYSize * src_trans[5]) + (round(
(sta[1] - (src_trans[3] - src_rast.RasterYSize * src_trans[5])) /
src_trans[5]) * src_trans[5])
te_str = str(x0) + " " + str(y0) + " " + str(x1) + " " + str(y1)
snowdays_ind_dst = path + "sta\\" + season + "\\" + ind + "_snowdays_" + season + ".tif"
if os.path.isfile(snowdays_ind_dst):
os.remove(snowdays_ind_dst)
#vector data from postgis
connString = "PG: host = " + conn_param.host + " dbname = " + conn_param.dbname + " user="******" password="******"select ind id, the_geom from stations.geo_dsa where ind = '" + sta[
4] + "'"
call(gdalwarp + " -co \"COMPRESS=LZW\" -co \"TILED=YES\" -cutline \"" +
connString + "\" -csql \"" + sql + "\" -te " + te_str +
" -dstnodata -9999 " + snow_file + " " + snowdays_ind_dst,
shell=True)
#create new tmp file (not tiled)... A voir.
format = "GTiff"
driver = gdal.GetDriverByName(format)
dst_file_tmp = path + "sta\\" + season + "\\viability_" + ind + "_tmp.tif"
if os.path.isfile(dst_file_tmp):
os.remove(dst_file_tmp)
dst_ds = driver.Create(dst_file_tmp, xsize, ysize, 3, gdal.GDT_Byte,
['TILED=YES', 'COMPRESS=LZW'])
dst_ds.SetGeoTransform(src_trans)
dst_ds.SetProjection(src_proj)
viable_index = numpy.zeros((ysize, xsize), numpy.uint8)
viable_index[numpy.isnan(mp_data)] = 1
# viability index
#col_off = int((src_trans[0] - snowdays_trans[0]) / snowdays_trans[1])
#row_off = int((snowdays_trans[3] - src_trans[3]) / -snowdays_trans[5])
snowdays_rast = None
snowdays_rast = gdal.Open(snowdays_ind_dst)
snowdays = snowdays_rast.GetRasterBand(1)
#snow = snowdays.ReadAsArray(col_off, row_off, xsize, ysize)
snow = snowdays.ReadAsArray()
viable_pix = mp_data[snow >= 100]
viability_index = numpy.sum(viable_pix[viable_pix > 0.])
#viable_index[snow >= 100] = 255
#print numpy.isfinite(mp_data), viable_index.shape, 218*194
dst_ds.GetRasterBand(3).WriteArray(viable_index)
viable_index = viable_index + 255 * numpy.logical_and(
numpy.isfinite(mp_data), snow >= 100)
dst_ds.GetRasterBand(2).WriteArray(viable_index)
viable_index[viable_index == 255] = 0
#viable_index.fill(0)
viable_index = viable_index + 255 * numpy.logical_and(
numpy.isfinite(mp_data), snow < 100)
dst_ds.GetRasterBand(1).WriteArray(viable_index)
dst_ds = None
snowdays_rast = None
dst_file = path + "sta\\" + season + "\\viability_" + ind + "_" + season + ".tif"
if os.path.isfile(dst_file):
os.remove(dst_file)
call(gdalwarp +
" -co \"COMPRESS=LZW\" -co \"TILED=YES\" -srcnodata 1 -dstnodata 1 " +
dst_file_tmp + " " + dst_file,
shell=True)
os.remove(dst_file_tmp)
query = "insert into stations.viability_index values(%s, %s, %s);"
viability.execute(query, (
ind,
year,
viability_index.tolist(),
))
myconn.commit()
#x[:,1] = lats[:]
#x[:,2] = no2[:]
nx = no2.shape[0]
ny = no2.shape[1]
#lons.shape[ny]
#xmin, ymin, xmax, ymax = [lons.min(), lats.min(), lons.max(), lats.max()]
xmin, ymin, xmax, ymax = -114.154,61.2847,49.3223,88.6367
#print xmin, ymin, xmax, ymax
xres = (xmax - xmin) / float(nx)
yres = (ymax - ymin) / float(ny)
geotransform = (xmin, xres, 0, ymax, 0, -yres)
#Creates 1 raster band file
#dst_ds = gdal.GetDriverByName('GTiff').Create('c:\\temp\data\myGeoTIFF.tif', ny, nx, 1, gdal.GDT_Float32)
dst_ds = gdal.GetDriverByName('GTiff').Create('c:\\temp\data\myGeoTIFF.tif', 450, 290, 1, gdal.GDT_Float32)
#
dst_ds.SetGeoTransform(geotransform) # specify coords
srs = osr.SpatialReference() # establish encoding
#srs.ImportFromEPSG(3857) # WGS84 lat/long
srs.ImportFromEPSG(4326) # WGS84 lat/long
dst_ds.SetProjection(srs.ExportToWkt()) # export coords to file
#dst_ds.GetRasterBand(1).WriteArray(np.squeeze(no2)) # write r-band to the raster
dst_ds.GetRasterBand(1).WriteArray(no2) # write r-band to the raster
dst_ds.FlushCache() # write to disk
dst_ds = None # save, close
def export_alos(arquivo, destino, refer, bino):
pos = arquivo.find(refer)
lat = -1*int(arquivo[pos+6:pos+8])
lon = -1*int(arquivo[pos+9:pos+12])
if arquivo[pos+5]=="N":
lat = lat*(-1)
#print "Norte:", lat
#else:
#print "Sul:", lat
#return 0
f = open(arquivo, "rb")
print f
count = 0
print os.stat(arquivo).st_size
npa = np.fromfile(f, dtype=np.dtype('<H'))
f.close()
npa = npa.astype(np.float32)
npa = npa**2
npr = npa.reshape(4500,4500)
#m81 = 0.8/81
#med81 = np.zeros((9,9))
#med81.fill(m81)
m25 = 1.0/25
med25 = np.zeros((5,5))
med25.fill(m25)
med9 = np.array([
[0.1,0.1,0.1],
[0.1,0.2,0.1],
[0.1,0.1,0.1],
])
#convolucao = ndimage.convolve(npr,med81)
convolucao = ndimage.convolve(npr,med25)
convolucao = ndimage.convolve(convolucao,med9)
convolucao = ndimage.convolve(convolucao,med9)
npalog = 10*np.log10(convolucao)-83
np.putmask(npalog, npalog<-500, -500)
npalog = npalog.reshape(4500,4500)
format = "GTiff"
driver = gdal.GetDriverByName( format )
metadata = driver.GetMetadata()
srs = osr.SpatialReference()
srs.SetWellKnownGeogCS( 'WGS84' )
dst_ds = driver.Create( destino, 4500, 4500, 1, gdal.GDT_Float32 )
dst_ds.SetGeoTransform( [ lon-0.000111111, 0.000222222, 0, lat-0.000111111, 0, -0.000222222 ] )
dst_ds.SetProjection( srs.ExportToWkt() )
dst_ds.GetRasterBand(1).WriteArray(npalog)
dst2_ds = driver.Create( destino.replace(".","_Class."), 4500, 4500, 1, gdal.GDT_Byte )
dst2_ds.SetGeoTransform( dst_ds.GetGeoTransform() )
dst2_ds.SetProjection( srs.ExportToWkt() )
np.putmask(npalog, npalog>-0.5, 0+bino)
np.putmask(npalog, npalog<-9.5, 0+bino)
np.putmask(npalog, npalog<0, 1-bino)
dst2_ds.GetRasterBand(1).WriteArray(npalog)
# Once we're done, close properly the dataset
dst_ds = None
print "Gerada imagem %s" %destino
def getRasterDriver(path, driver_name=None):
return gdal.GetDriverByName(guessRasterDriver(path))
def get_angle(view_ang_name_gml, vaa, vza, band_dict):
band_name, view_ang_name, gml = view_ang_name_gml
g = ogr.Open(gml)
xRes = 10
yRes = 10
g1 = gdal.Open(band_name)
geo_t = g1.GetGeoTransform()
x_size, y_size = g1.RasterXSize, g1.RasterYSize
vas = np.zeros((y_size, x_size), dtype=np.float32)
vas[:] = -327.67
vzs = np.zeros((y_size, x_size), dtype=np.float32)
vzs[:] = -327.67
x_min, x_max = min(geo_t[0], geo_t[0] + x_size * geo_t[1]), \
max(geo_t[0], geo_t[0] + x_size * geo_t[1])
y_min, y_max = min(geo_t[3], geo_t[3] + y_size * geo_t[5]), \
max(geo_t[3], geo_t[3] + y_size * geo_t[5])
xRes, yRes = abs(geo_t[1]), abs(geo_t[5])
x_scale = 5000. / xRes
y_scale = 5000. / yRes
layer = g.GetLayer()
foot1 = None
foot2 = None
va1 = None
vz1 = None
va2 = None
vz2 = None
try:
dets = []
for i in range(layer.GetFeatureCount()):
dets.append(layer.GetFeature(i).items()['gml_id'])
dets = sorted(dets, key=lambda i: int(i.split('-')[2]))
for i in range(len(dets)):
det = dets[i]
foot1 = gdal.Rasterize("", gml, format="MEM", xRes=xRes, yRes=yRes, \
where="gml_id='%s'"%det, outputBounds=[ x_min, y_min, x_max, y_max], \
noData=0, burnValues=1, outputType=gdal.GDT_Byte).ReadAsArray()
foot1 = binary_dilation(foot1)
key = band_dict[det.split('-')[-3]], int(det.split('-')[-2])
va1 = vaa[key]
vz1 = vza[key]
if i > 0:
overlap = foot1 * foot2
if overlap.sum() < 10:
foot1 = foot2
else:
x, y = np.where(overlap)
xmin, xmax, ymin, ymax = x.min(), x.max(), y.min(), y.max()
ll = x[x == xmax][-1], y[x == xmax][-1]
lr = x[y == ymax][-1], y[y == ymax][-1]
ul = x[y == ymin][0], y[y == ymin][0]
ur = x[x == xmin][0], y[x == xmin][0]
p1 = np.mean([lr, ur], axis=0)
p2 = np.mean([ll, ul], axis=0)
x1, y1 = np.where(foot2)
vamax, vamin = np.nanmax(va2), np.nanmin(va2)
vzmax, vzmin = np.nanmax(vz2), np.nanmin(vz2)
if not (p1 == p2).all():
p = np.poly1d(
np.polyfit([p1[1], p2[1]], [p1[0], p2[0]], 1))
foot2[x[x > p(y)], y[x > p(y)]] = False
minx, miny = np.where(va2 == vamin)
maxx, maxy = np.where(va2 == vamax)
min_dst = abs(p(miny * y_scale) - minx * x_scale) / (
np.sqrt(1 + p.c[0]**2))
max_dst = abs(p(maxy * y_scale) - maxx * x_scale) / (
np.sqrt(1 + p.c[0]**2))
if (max_dst < min_dst).any():
tmp1 = vamin.copy()
vamin = vamax
vamax = tmp1
minx, miny = np.where(vz2 == vzmin)
maxx, maxy = np.where(vz2 == vzmax)
min_dst = abs(p(miny * y_scale) - minx * x_scale) / (
np.sqrt(1 + p.c[0]**2))
max_dst = abs(p(maxy * y_scale) - maxx * x_scale) / (
np.sqrt(1 + p.c[0]**2))
if (max_dst < min_dst).any():
tmp2 = vzmin.copy()
vzmin = vzmax
vzmax = tmp2
dist = abs(p(y1) - x1) / (np.sqrt(1 + p.c[0]**2))
vas[x1,
y1] = vamin + dist / (dist.max() -
dist.min()) * (vamax - vamin)
vzs[x1,
y1] = vzmin + dist / (dist.max() -
dist.min()) * (vzmax - vzmin)
else:
vas[x1, y1] = vamin
vzs[x1, y1] = vzmin
x1, y1 = np.where(foot1)
if i == layer.GetFeatureCount() - 1:
vamax, vamin = np.nanmax(va1), np.nanmin(va1)
vzmax, vzmin = np.nanmax(vz1), np.nanmin(vz1)
if not (p1 == p2).all():
foot1[x[x <= p(y)], y[x <= p(y)]] = False
minx, miny = np.where(va1 == vamin)
maxx, maxy = np.where(va1 == vamax)
min_dst = abs(p(miny * y_scale) - minx *
x_scale) / (np.sqrt(1 + p.c[0]**2))
max_dst = abs(p(maxy * y_scale) - maxx *
x_scale) / (np.sqrt(1 + p.c[0]**2))
if (max_dst < min_dst).any():
tmp1 = vamin.copy()
vamin = vamax
vamax = tmp1
minx, miny = np.where(vz1 == vzmin)
maxx, maxy = np.where(vz1 == vzmax)
min_dst = abs(p(miny * y_scale) - minx *
x_scale) / (np.sqrt(1 + p.c[0]**2))
max_dst = abs(p(maxy * y_scale) - maxx *
x_scale) / (np.sqrt(1 + p.c[0]**2))
if (max_dst < min_dst).any():
tmp2 = vzmin.copy()
vzmin = vzmax
vzmax = tmp2
dist = abs(p(y1) - x1) / (np.sqrt(1 + p.c[0]**2))
vas[x1, y1] = vamin + dist / (
dist.max() - dist.min()) * (vamax - vamin)
vzs[x1, y1] = vzmin + dist / (
dist.max() - dist.min()) * (vzmax - vzmin)
else:
vas[x1, y1] = vamin
vas[x1, y1] = vamin
foot2 = foot1
va2 = va1
vz2 = vz1
# vas[:] = np.nanmean(vaa.values())
# vzs[:] = np.nanmean(vza.values())
if os.path.exists(view_ang_name):
os.remove(view_ang_name)
dst_ds = gdal.GetDriverByName('GTiff').Create(
view_ang_name,
x_size,
y_size,
2,
gdal.GDT_Int16,
options=["TILED=YES", "COMPRESS=DEFLATE"])
dst_ds.SetGeoTransform(g1.GetGeoTransform())
dst_ds.SetProjection(g1.GetProjection())
mask = vas < -180
if (~mask).sum() < 1:
vas[:] = np.nanmean(va1)
#vas = fill_bad(vas, ~mask)
mask = vzs < 0
if (~mask).sum() < 1:
vzs[:] = np.nanmean(vz1)
#vzs = fill_bad(vzs, ~mask)
# azimuth angle is very unstable so only chaning is at 180
vas[(vas > 180)
& (vas <= 360)] = vas[(vas > 180) & (vas <= 360)].mean() - 360
vas[(vas >= 0) & (vas <= 180)] = vas[(vas >= 0) & (vas <= 180)].mean()
dst_ds.GetRasterBand(1).WriteArray((vas * 100).astype(int))
dst_ds.GetRasterBand(2).WriteArray((vzs * 100).astype(int))
dst_ds.GetRasterBand(1).SetNoDataValue(-32767)
dst_ds.GetRasterBand(2).SetNoDataValue(-32767)
dst_ds.FlushCache()
dst_ds = None
g1 = None
return True
except:
return False
import ogr, osr
import gdal
###
###
###
# Draw Polygones with gdal: thx to https://gis.stackexchange.com/a/200634
# Setup working spatial reference
sr_wkt = 'LOCAL_CS["arbitrary"]'
sr = osr.SpatialReference()
sr.SetWellKnownGeogCS('WGS84')
# Create a memory raster to rasterize into.
target_ds = gdal.GetDriverByName('MEM').Create('', 200, 200, 3, gdal.GDT_Byte)
# https://gis.stackexchange.com/questions/165950/gdal-setgeotransform-does-not-work-as-expected
# target_ds.SetGeoTransform((1000, 1, 0, 1100, 0, -1))
target_ds.SetGeoTransform((9.948, 1, 0, 49.7600, 0, -1))
sr.ImportFromEPSG(3857)
# Create a memory layer to rasterize from.
rast_ogr_ds = \
ogr.GetDriverByName('Memory').CreateDataSource('wrk')
rast_mem_lyr = rast_ogr_ds.CreateLayer('poly', srs=sr)
# Add a polygon with hole.
wkt_geom = 'POLYGON((1020 1076 ,1025 1085 ,1065 1090 ,1064 1078 ,1020 1076 ), (1023 1079 ,1061 1081 ,1062 1087 ,1028 1082 ,1023 1079 ))'
with open("file.json") as ft:
data = ft.read()
mygeo = ogr.CreateGeometryFromJson(data)
def parse_xml(meta_file, example_file, sun_ang_name):
tree = ET.parse(meta_file)
root = tree.getroot()
saa = []
sza = []
msz = []
msa = []
vza = {}
vaa = {}
mvz = {}
mva = {}
for child in root:
for j in child:
for k in j.findall('Sun_Angles_Grid'):
for l in k.findall('Zenith'):
for m in l.findall('Values_List'):
for x in m.findall('VALUES'):
sza.append(x.text.split())
for n in k.findall('Azimuth'):
for o in n.findall('Values_List'):
for p in o.findall('VALUES'):
saa.append(p.text.split())
for ms in j.findall('Mean_Sun_Angle'):
msz = float(ms.find('ZENITH_ANGLE').text)
msa = float(ms.find('AZIMUTH_ANGLE').text)
for k in j.findall('Viewing_Incidence_Angles_Grids'):
for l in k.findall('Zenith'):
for m in l.findall('Values_List'):
vza_sub = []
for x in m.findall('VALUES'):
vza_sub.append(x.text.split())
bi, di, angles = k.attrib['bandId'], \
k.attrib['detectorId'], np.array(vza_sub).astype(float)
vza[(int(bi), int(di))] = angles
for n in k.findall('Azimuth'):
for o in n.findall('Values_List'):
vaa_sub = []
for p in o.findall('VALUES'):
vaa_sub.append(p.text.split())
bi, di, angles = k.attrib['bandId'],\
k.attrib['detectorId'], np.array(vaa_sub).astype(float)
vaa[(int(bi), int(di))] = angles
for mvia in j.findall('Mean_Viewing_Incidence_Angle_List'):
for i in mvia.findall('Mean_Viewing_Incidence_Angle'):
mvz[int(i.attrib['bandId'])] = float(
i.find('ZENITH_ANGLE').text)
mva[int(i.attrib['bandId'])] = float(
i.find('AZIMUTH_ANGLE').text)
sza = np.array(sza).astype(float)
saa = np.array(saa).astype(float)
saa[saa > 180] = saa[saa > 180] - 360
g = gdal.Open(example_file)
geo = g.GetGeoTransform()
projection = g.GetProjection()
geotransform = (geo[0], 5000, geo[2], geo[3], geo[4], -5000)
if os.path.exists(sun_ang_name):
os.remove(sun_ang_name)
dst_ds = gdal.GetDriverByName('GTiff').Create(
sun_ang_name,
23,
23,
2,
gdal.GDT_Int16,
options=["TILED=YES", "COMPRESS=DEFLATE"])
dst_ds.SetGeoTransform(geotransform)
dst_ds.SetProjection(projection)
dst_ds.GetRasterBand(1).WriteArray((saa * 100).astype(int))
dst_ds.GetRasterBand(2).WriteArray((sza * 100).astype(int))
dst_ds, g = None, None
return vaa, vza
if j%100==0:
print(j)
#print(j) uncomment to monitor predicing stages
sam_row = c1.all_sample_row(j)
if patch == 7:
sam_row = sam_row.reshape(sam_row.shape[0],patch,patch,im1z,1)
pre_row1 = np.argmax(model1.predict(sam_row),axis=1)
pre_row1 = pre_row1.reshape(1,im1y)
pre_rows_1.append(pre_row1)
pre_all_1.append(np.array(pre_rows_1))
time4 = int(time.time())
print('predict time:',time4-time3) # predict time
# classification map and confusion matrix for raw classification
pre_all_1 = np.array(pre_all_1).reshape(ensemble,im1x,im1y)
pre1 = np.int8(stats.mode(pre_all_1,axis=0)[0]).reshape(im1x,im1y)
result11 = rscls.gtcfm(pre1+1,c1.gt+1,cls1)
saved[str(seedi)+'a'] = result11
rscls.save_cmap(pre1, 'jet', 'pre.png')
# save as geocode-tif
name = 'predict_'+str(time4)[-5:]
outdata = gdal.GetDriverByName('GTiff').Create(name+'.tif', im1y, im1x, 1, gdal.GDT_UInt16)
outdata.SetGeoTransform(newgeo)
outdata.SetProjection(projection)
outdata.GetRasterBand(1).WriteArray(pre1+1)
outdata.FlushCache() ##saves to disk!!
outdata = None
def SaveArray(src_array, filename, format="GTiff", prototype=None):
driver = gdal.GetDriverByName(format)
if driver is None:
raise ValueError("Can't find driver " + format)
return driver.CreateCopy(filename, OpenArray(src_array, prototype))
def snow_season(src_loc, snow_dst_file, year):
myconn = psycopg2.connect("host=" + conn_param.host + " dbname=" +
conn_param.dbname + " user="******" password="******"-" + str(year + 1)
loc_rast = gdal.Open(src_loc)
loc_band = loc_rast.GetRasterBand(1)
#Get metadata
xsize = loc_band.XSize
ysize = loc_band.YSize
block_sizes = loc_band.GetBlockSize()
x_block_size = block_sizes[0]
y_block_size = block_sizes[1]
max_value = loc_band.GetMaximum()
min_value = loc_band.GetMinimum()
if max_value == None or min_value == None:
stats = loc_band.GetStatistics(0, 1)
max_value = stats[1]
min_value = stats[0]
trans = loc_rast.GetGeoTransform()
proj = loc_rast.GetProjection()
format = "GTiff"
driver = gdal.GetDriverByName(format)
if os.path.isfile(snow_dst_file):
os.remove(snow_dst_file)
snow = driver.Create(snow_dst_file, xsize, ysize, 1, gdal.GDT_Int16,
['TILED=YES', 'COMPRESS=LZW'])
snow.SetGeoTransform(trans)
snow.SetProjection(proj)
snow.SetNoDataValue(-9999)
for i in range(0, ysize, y_block_size):
if i + y_block_size < ysize:
rows = y_block_size
else:
rows = ysize - i
for j in range(0, xsize, x_block_size):
if j + x_block_size < xsize:
cols = x_block_size
else:
cols = xsize - j
data_loc = loc_band.ReadAsArray(j, i, cols, rows)
unique_loc = numpy.unique(data_loc)
loc_list = ', '.join(map(str, unique_loc))
begin = str(year) + "-11-01"
end = str(year + 1) + "-04-30"
snowdays = myconn.cursor()
query = ("""
select loc, count(crocus_date) nb_days from stations.meteo_crocus
where loc in (select unnest(string_to_array(%s,', '))::integer)
and crocus_date between %s and %s
and hauteur_neige >= 0.3
group by 1
order by 1
""")
snowdays.execute(query, (
loc_list,
begin,
end,
))
if snowdays.rowcount == 0:
nb_days = numpy.zeros((rows, cols), numpy.int16)
nb_days[data_loc == 0] = -9999
snow.GetRasterBand(1).WriteArray(nb_days, j, i)
print i, j, "no test"
else:
l = 0
snow_days = numpy.zeros((snowdays.rowcount, 2), numpy.int16)
for days in snowdays:
for m in range(0, 2):
snow_days[l, m] = days[m]
l = l + 1
nb_days = numpy.zeros((rows, cols), numpy.int16)
for k in range(0, snow_days.shape[0]):
nb_days = nb_days + snow_days[k, 1] * (data_loc
== snow_days[k, 0])
nb_days[data_loc == 0] = -9999
print i, j
snow.GetRasterBand(1).WriteArray(nb_days, j, i)
snow.BuildOverviews("NEAREST", 6, {2, 4, 8, 16, 32, 64})
snow = None
def Collect_data(TilesHorizontal,TilesVertical,Date,output_folder, hdf_library):
'''
This function downloads all the needed MODIS tiles from http://e4ftl01.cr.usgs.gov/MOLT/MOD13Q1.006/ as a hdf file.
Keywords arguments:
TilesHorizontal -- [TileMin,TileMax] max and min horizontal tile number
TilesVertical -- [TileMin,TileMax] max and min vertical tile number
Date -- 'yyyy-mm-dd'
output_folder -- 'C:/file/to/path/'
'''
# Make a new tile for the data
sizeX = int((TilesHorizontal[1] - TilesHorizontal[0] + 1) * 4800)
sizeY = int((TilesVertical[1] - TilesVertical[0] + 1) * 4800)
DataTot = np.zeros((sizeY, sizeX))
# Load accounts
username, password = WebAccounts.Accounts(Type = 'NASA')
# Create the Lat and Long of the MODIS tile in meters
for Vertical in range(int(TilesVertical[0]), int(TilesVertical[1])+1):
Distance = 231.65635826395834 # resolution of a MODIS pixel in meter
countY=(TilesVertical[1] - TilesVertical[0] + 1) - (Vertical - TilesVertical[0])
for Horizontal in range(int(TilesHorizontal[0]), int(TilesHorizontal[1]) + 1):
countX=Horizontal - TilesHorizontal[0] + 1
# Download the MODIS NDVI data
url = 'https://e4ftl01.cr.usgs.gov/MOLT/MOD09GQ.006/' + Date.strftime('%Y') + '.' + Date.strftime('%m') + '.' + Date.strftime('%d') + '/'
# Reset the begin parameters for downloading
downloaded = 0
N=0
# Check the library given by user
if hdf_library is not None:
os.chdir(hdf_library)
hdf_name = glob.glob("MOD09GQ.A%s%03s.h%02dv%02d.*" %(Date.strftime('%Y'), Date.strftime('%j'), Horizontal, Vertical))
if len(hdf_name) == 1:
hdf_file = os.path.join(hdf_library, hdf_name[0])
if os.path.exists(hdf_file):
downloaded = 1
file_name = hdf_file
if not downloaded == 1:
# Get files on FTP server
f = urllib2.urlopen(url)
# Sum all the files on the server
soup = BeautifulSoup(f, "lxml")
for i in soup.findAll('a', attrs = {'href': re.compile('(?i)(hdf)$')}):
# Find the file with the wanted tile number
Vfile=str(i)[30:32]
Hfile=str(i)[27:29]
if int(Vfile) is int(Vertical) and int(Hfile) is int(Horizontal):
# Define the whole url name
full_url = urlparse.urljoin(url, i['href'])
# if not downloaded try to download file
while downloaded == 0:
try:# open http and download whole .hdf
nameDownload = full_url
file_name = os.path.join(output_folder,nameDownload.split('/')[-1])
if os.path.isfile(file_name):
downloaded = 1
else:
x = requests.get(nameDownload, allow_redirects = False)
try:
y = requests.get(x.headers['location'], auth = (username, password))
except:
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
y = requests.get(x.headers['location'], auth = (username, password), verify = False)
z = open(file_name, 'wb')
z.write(y.content)
z.close()
statinfo = os.stat(file_name)
# Say that download was succesfull
if int(statinfo.st_size) > 10000:
downloaded = 1
# If download was not succesfull
except:
# Try another time
N = N + 1
# Stop trying after 10 times
if N == 10:
print 'Data from ' + Date.strftime('%Y-%m-%d') + ' is not available'
downloaded = 1
try:
# Open .hdf only band with NDVI and collect all tiles to one array
dataset = gdal.Open(file_name)
sdsdict = dataset.GetMetadata('SUBDATASETS')
sdslist = [sdsdict[k] for k in sdsdict.keys() if '_2_NAME' in k]
sds = []
for n in sdslist:
sds.append(gdal.Open(n))
full_layer = [i for i in sdslist if 'sur_refl_b01_1' in i]
idx = sdslist.index(full_layer[0])
if Horizontal == TilesHorizontal[0] and Vertical == TilesVertical[0]:
geo_t = sds[idx].GetGeoTransform()
# get the projection value
proj = sds[idx].GetProjection()
data = sds[idx].ReadAsArray()
countYdata = (TilesVertical[1] - TilesVertical[0] + 2) - countY
DataTot[int((countYdata - 1) * 4800):int(countYdata * 4800), int((countX - 1) * 4800):int(countX * 4800)]=data
del data
# if the tile not exists or cannot be opened, create a nan array with the right projection
except:
if Horizontal==TilesHorizontal[0] and Vertical==TilesVertical[0]:
x1 = (TilesHorizontal[0] - 19) * 4800 * Distance
x4 = (TilesVertical[0] - 9) * 4800 * -1 * Distance
geo = [x1, Distance, 0.0, x4, 0.0, -Distance]
geo_t=tuple(geo)
proj='PROJCS["unnamed",GEOGCS["Unknown datum based upon the custom spheroid",DATUM["Not specified (based on custom spheroid)",SPHEROID["Custom spheroid",6371007.181,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Sinusoidal"],PARAMETER["longitude_of_center",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1]]'
data=np.ones((4800,4800)) * (-9999)
countYdata=(TilesVertical[1] - TilesVertical[0] + 2) - countY
DataTot[(countYdata - 1) * 4800:countYdata * 4800,(countX - 1) * 4800:countX * 4800] = data
# Make geotiff file
name2 = os.path.join(output_folder, 'Merged.tif')
driver = gdal.GetDriverByName("GTiff")
dst_ds = driver.Create(name2, DataTot.shape[1], DataTot.shape[0], 1, gdal.GDT_Float32, ['COMPRESS=LZW'])
try:
dst_ds.SetProjection(proj)
except:
proj='PROJCS["unnamed",GEOGCS["Unknown datum based upon the custom spheroid",DATUM["Not specified (based on custom spheroid)",SPHEROID["Custom spheroid",6371007.181,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Sinusoidal"],PARAMETER["longitude_of_center",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1]]'
x1 = (TilesHorizontal[0] - 18) * 4800 * Distance
x4 = (TilesVertical[0] - 9) * 4800 * -1 * Distance
geo = [x1, Distance, 0.0, x4, 0.0, -Distance]
geo_t = tuple(geo)
dst_ds.SetProjection(proj)
dst_ds.GetRasterBand(1).SetNoDataValue(-9999)
dst_ds.SetGeoTransform(geo_t)
dst_ds.GetRasterBand(1).WriteArray(DataTot*0.0001)
dst_ds = None
sds = None
return()
def mp_resort_rast(path, ind):
gdalwarp = "C:\Python276\Lib\site-packages\osgeo\gdalwarp.exe"
#psycopg2 connection to DB
myconn = psycopg2.connect("host=" + conn_param.host + " dbname=" +
conn_param.dbname + " user="******" password="******"""
with a as (
select ind, st_envelope(st_buffer(the_geom,25)) geom
from stations.geo_dsa
where ind = %s
order by 1
)
select st_xmin(geom), st_ymin(geom), st_xmax(geom), st_ymax(geom), ind from a
"""
cur.execute(query, (ind, ))
#new rasters extent
src_crocus_alt = path + "crocus_altitude.tif"
src_rast = gdal.Open(src_crocus_alt)
src_trans = src_rast.GetGeoTransform()
src_crocus_slope = path + "crocus_slope.tif"
src_crocus_aspect = path + "crocus_aspect.tif"
sta = cur.fetchone()
print sta[4]
x0 = src_trans[0] + (round(
(sta[0] - src_trans[0]) / src_trans[1]) * src_trans[1])
x1 = (src_trans[0] + src_rast.RasterXSize * src_trans[1]) - (round(
((src_trans[0] + src_rast.RasterXSize * src_trans[1]) - sta[2]) /
src_trans[1]) * src_trans[1])
y1 = src_trans[3] - (round(
(src_trans[3] - sta[3]) / src_trans[5]) * src_trans[5])
y0 = (src_trans[3] - src_rast.RasterYSize * src_trans[5]) + (round(
(sta[1] - (src_trans[3] - src_rast.RasterYSize * src_trans[5])) /
src_trans[5]) * src_trans[5])
te_str = str(x0) + " " + str(y0) + " " + str(x1) + " " + str(y1)
dst_file = path + "sta\\crocus_alt_" + sta[4] + ".tif"
dst_file_slope = path + "sta\\crocus_slope_" + sta[4] + ".tif"
dst_file_aspect = path + "sta\\crocus_aspect_" + sta[4] + ".tif"
if os.path.isfile(dst_file):
os.remove(dst_file)
newdir = path + "sta"
if os.path.isdir(newdir) == False:
os.mkdir(newdir)
#vector data from postgis
connString = "PG: host = " + conn_param.host + " dbname = " + conn_param.dbname + " user="******" password="******"select ind id, the_geom from stations.geo_dsa where ind = '" + sta[
4] + "'"
#cutting and prepare data to build resort mp raster
call(gdalwarp + " -co \"COMPRESS=LZW\" -co \"TILED=YES\" -cutline \"" +
connString + "\" -csql \"" + sql + "\" -te " + te_str +
" -dstnodata -9999 " + src_crocus_alt + " " + dst_file,
shell=True)
call(gdalwarp + " -co \"COMPRESS=LZW\" -co \"TILED=YES\" -cutline \"" +
connString + "\" -csql \"" + sql + "\" -te " + te_str +
" -dstnodata -9999 " + src_crocus_slope + " " + dst_file_slope,
shell=True)
call(gdalwarp + " -co \"COMPRESS=LZW\" -co \"TILED=YES\" -cutline \"" +
connString + "\" -csql \"" + sql + "\" -te " + te_str +
" -dstnodata -9999 " + src_crocus_aspect + " " + dst_file_aspect,
shell=True)
alt_rast = gdal.Open(dst_file)
trans = alt_rast.GetGeoTransform()
proj = alt_rast.GetProjection()
alt_band = alt_rast.GetRasterBand(1)
alt_arr = alt_band.ReadAsArray()
xsize = alt_band.XSize
ysize = alt_band.YSize
format = "GTiff"
driver = gdal.GetDriverByName(format)
dst_file = "C:\ds_test_data\snow\sta\\" + sta[4] + ".tif"
if os.path.isfile(dst_file):
os.remove(dst_file)
dst_ds = driver.Create(dst_file, xsize, ysize, 1, gdal.GDT_Float32)
dst_ds.SetGeoTransform(trans)
dst_ds.SetProjection(proj)
mp_data = numpy.zeros((ysize, xsize), numpy.float32)
#get alt and mp_part for sta
ski_data = myconn.cursor()
query = """
select alti_crocus, mp, mp_tot from stations.geo_enveloppes_rm_alpes_alti_crocus
where indicatif_station = %s;"""
ski_data.execute(query, (sta[4], ))
for ski in ski_data:
nb_pix = (alt_arr == ski[0]).sum()
if nb_pix != 0:
pix_val = (float(ski[1]) / nb_pix / ski[2]) * 100
mp_data[alt_arr == ski[0]] = (pix_val)
mp_data[mp_data == 0] = numpy.nan
dst_ds.GetRasterBand(1).WriteArray(mp_data)
dst_ds = None
os.chdir('c:/temp')
data2 = Numeric.array([ [0,54,100], [87,230,5], [161,120,24] ])
data3 = Numeric.array([ [0,100,23], [78,29,1], [134,245,0] ])
print data2
print data3
ndvi = (data3 - data2) / (data3 + data2)
mask = Numeric.greater(data3 + data2, 0)
print mask
ndvi = Numeric.choose(mask, (-99, (data3 - data2) / (data3 + data2)))
data3 = data3.astype(Numeric.Float16)
data2 = data2.astype(Numeric.Float16)
ndvi = Numeric.choose(mask, (-99, (data3 - data2) / (data3 + data2)))
print ndvi
import gdal
from gdalconst import *
driver = gdal.GetDriverByName('HFA')
ds = driver.Create('sample1.img', 3, 3, 1, GDT_Float32)
band = ds.GetRasterBand(1)
band.WriteArray(ndvi, 0, 0)
ds = None
ds = driver.Create('sample2.img', 3, 3, 1, GDT_Float32)
band = ds.GetRasterBand(1)
band.WriteArray(ndvi, 0, 0)
ds = None
ds = driver.Create('sample3.img', 3, 3, 1, GDT_Float32)
band = ds.GetRasterBand(1)
band.WriteArray(ndvi, 0, 0)
band.FlushCache()
band.SetNoDataValue(-99)
band.GetStatistics(0,1)
ds = None
def get_location(path):
dst_crocus_alt = path + "crocus_altitude.tif"
dst_crocus_slope = path + "crocus_slope.tif"
dst_crocus_aspect = path + "crocus_aspect.tif"
dst_massif = path + "crocus_massifs_safran.tif"
#Opening mandatory rasters
alt_rast = gdal.Open(dst_crocus_alt)
alt_band = alt_rast.GetRasterBand(1)
slope_rast = gdal.Open(dst_crocus_slope)
slope_band = slope_rast.GetRasterBand(1)
aspect_rast = gdal.Open(dst_crocus_aspect)
aspect_band = aspect_rast.GetRasterBand(1)
massif_rast = gdal.Open(dst_massif)
massif_band = massif_rast.GetRasterBand(1)
#Get metadata
xsize = alt_band.XSize
ysize = alt_band.YSize
block_sizes = alt_band.GetBlockSize()
x_block_size = block_sizes[0]
y_block_size = block_sizes[1]
max_value = alt_band.GetMaximum()
min_value = alt_band.GetMinimum()
if max_value == None or min_value == None:
stats = alt_band.GetStatistics(0, 1)
max_value = stats[1]
min_value = stats[0]
trans = alt_rast.GetGeoTransform()
proj = alt_rast.GetProjection()
format = "GTiff"
driver = gdal.GetDriverByName(format)
dst_file = path + "crocus_location_tmp.tif"
if os.path.isfile(dst_file):
os.remove(dst_file)
dst_ds = driver.Create(dst_file, xsize, ysize, 1, gdal.GDT_Int16)
dst_ds.SetGeoTransform(trans)
dst_ds.SetProjection(proj)
###################BLOCK VERSION
for i in range(0, ysize, y_block_size):
if i + y_block_size < ysize:
rows = y_block_size
else:
rows = ysize - i
for j in range(0, xsize, x_block_size):
if j + x_block_size < xsize:
cols = x_block_size
else:
cols = xsize - j
data_alt = alt_band.ReadAsArray(j, i, cols, rows)
data_slope = slope_band.ReadAsArray(j, i, cols, rows)
data_aspect = aspect_band.ReadAsArray(j, i, cols, rows)
data_massif = massif_band.ReadAsArray(j, i, cols, rows)
unique_alt = numpy.unique(data_alt)
alt_list = ', '.join(map(str, unique_alt))
unique_slope = numpy.unique(data_slope)
slope_list = ', '.join(map(str, unique_slope))
unique_aspect = numpy.unique(data_aspect)
aspect_list = ', '.join(map(str, unique_aspect))
unique_massif = numpy.unique(data_massif)
massif_list = ', '.join(map(str, unique_massif))
#get loc query
location = myconn.cursor()
query = ("""
with
ref as (
select distinct loc::int, alti::int, slope::int, aspect::int, ref_massif_meteo::int from stations.meteo_crocus_location a
join stations.passage_meteo_massif_ind b on a.ref_point_crocus = b.ref_point_crocus
),
a as (select * from ref where alti in (select unnest(string_to_array(%s,', '))::integer)),
b as (select * from a where slope in (select unnest(string_to_array(%s,', '))::integer) and slope != 0),
c as (select * from b where aspect in (select unnest(string_to_array(%s,', '))::integer))
select * from c where ref_massif_meteo in (select unnest(string_to_array(%s,', '))::integer)
""")
location.execute(query, (
alt_list,
slope_list,
aspect_list,
massif_list,
))
if location.rowcount == 0:
print i, j, "no test"
else:
l = 0
crocus_loc = numpy.zeros((location.rowcount, 5), numpy.int16)
for loc in location:
for m in range(0, 5):
crocus_loc[l, m] = loc[m]
l = l + 1
loc_output = numpy.zeros((rows, cols), numpy.int16)
for k in range(0, crocus_loc.shape[0]):
#print i,j,k
test1 = numpy.logical_and(data_alt == crocus_loc[k, 1],
data_slope == crocus_loc[k, 2])
test2 = numpy.logical_and(data_aspect == crocus_loc[k, 3],
data_massif == crocus_loc[k, 4])
loc_output = loc_output + crocus_loc[
k, 0] * numpy.logical_and(test1, test2)
#slope = 0 and no aspect
query = ("""
with
ref as (
select distinct loc::int, alti::int, slope::int, aspect::int, ref_massif_meteo::int from stations.meteo_crocus_location a
join stations.passage_meteo_massif_ind b on a.ref_point_crocus = b.ref_point_crocus
),
a as (select * from ref where alti in (select unnest(string_to_array(%s,', '))::integer)),
b as (select * from a where slope = 0)
select * from b where ref_massif_meteo in (select unnest(string_to_array(%s,', '))::integer)
""")
location.execute(query, (
alt_list,
massif_list,
))
if location.rowcount == 0:
print "no null aspect test"
else:
l = 0
crocus_loc = numpy.zeros((location.rowcount, 5),
numpy.int16)
for loc in location:
for m in range(0, 5):
crocus_loc[l, m] = loc[m]
l = l + 1
for k in range(0, crocus_loc.shape[0]):
#print i,j,k
test1 = numpy.logical_and(data_alt == crocus_loc[k, 1],
data_slope == 0)
loc_output = loc_output + crocus_loc[
k, 0] * numpy.logical_and(
test1, data_massif == crocus_loc[k, 4])
dst_ds.GetRasterBand(1).WriteArray(loc_output, j, i)
dst_ds = None
dst_file_tiled = path + "crocus_location.tif"
if os.path.isfile(dst_file_tiled):
os.remove(dst_file_tiled)
call(gdalwarp + " -co \"COMPRESS=LZW\" -co \"TILED=YES\" " + dst_file +
" " + dst_file_tiled,
shell=True)
def rast_math(output_path, expression, *args):
"""
A raster math calculator that uses GDALs python bindings instead of command line
interface for simple math. The syntax of this feels more pythonic than the native gdal_calc.py
syntax. Supports up to 26 raster images for each letter of the alphabet as
arguments. Supports single band raster images or gdal.Band instances as *args inputs.
Input expression will be directly evaluated, so users can input numerical constants
and simple ``numpy`` or ``math`` module operators with lowercase function names. Because
this function uses python bindings and numpy data structures, be mindful of the memory
limitations associated with 32-bit python, highly recommend using 64 bit.
:param output_path: filepath at which to store expression result. Set to False to just return
the numeric array instead of saving it.
:param expression: the mathematical expression using rasters as A,B,C, etc
:param args: Filepaths to single band rasters that represent A,B,C, etc (in order)
OR, gdal.Band instances which could be used with multi-band rasters via...
ds = gdal.Open(rastpath)
bandA = ds.GetRasterBand(1)
bandB = ds.GetRasterBand(2)
rast_math(outpath, "numpy.log(A) + 3 * B", bandA, bandB)
:return: the output path to the file created by this function
An example for the ubiquitous NDVI calculation from landsat bands:
..code-block: python
root = r"my_landsat_directory" # directory with landsat tiffs
A_path = os.path.join(root, "tile_B5.TIF") # filepath to Band5
B_path = os.path.join(root, "tile_B4.TIF") # filepath to Band4
out_path = os.path.join(root, "NDVI.TIF") # filepath of new output image
rast_math(out_path, "(A + B) / (A - B)", A_path, B_path)
An example where we want to conditionally mask one raster by another, say
image "A" is our raster with data, and image "B" is a mask where a value of 1
indicates a bad value, and zero indicates a good value.
..code-block:python
rast_math(out_path, "A * (B == 0)", A_path, B_path)
"""
# set up the iterators and structures
datasets = {} # dictionary where actual data will go
eval_args = {} # dictionary with string literals to be evaluated as code
alphabet = string.ascii_uppercase[:len(args)]
# format the expression with curly brackets around letters
print("Executing expression '{0}'".format(expression))
for letter in alphabet:
expression = expression.replace(letter, "{%s}" % letter)
# create the numpy arrays from raster datasets with gdal
for arg, letter in zip(args, alphabet):
# handle filepath input and raise exception for invalid filepaths
if isinstance(arg, str):
if not os.path.exists(arg):
raise Exception("file {0} does not exist!".format(arg))
print("\tLoading {0} as raster '{1}'".format(arg, letter))
dataset_in = gdal.Open(arg, gdalconst.GA_ReadOnly)
band = dataset_in.GetRasterBand(1)
datasets[letter] = numpy.array(band.ReadAsArray(), dtype="float32")
# handles input type of a gdal.Band instance
elif isinstance(arg, gdal.Band):
datasets[letter] = numpy.array(arg.ReadAsArray(), dtype="float32")
eval_args[letter] = "datasets['{0}']".format(letter)
# assemble and evaluate the expression
eval_expression = expression.format(**eval_args)
print(eval_expression)
out_array = eval(eval_expression)
# either save the output or return an output array
if output_path:
driver = gdal.GetDriverByName(
"GTiff") # create the geotiff driver object
yshape, xshape = datasets["A"].shape # set dimensions of output file
num_bands = 1 # only supports single band output
dataset_out = driver.Create(output_path, xshape, yshape, num_bands,
gdal.GDT_Float32)
gdalnumeric.CopyDatasetInfo(dataset_in, dataset_out)
band_out = dataset_out.GetRasterBand(1)
gdalnumeric.BandWriteArray(band_out, out_array)
return os.path.abspath(output_path)
else:
return out_array
def Bin2GeoTiff(infile, outfilepath):
'''
This function takes the NSIDC charts, being a flat binary string, and converts them to GeoTiff. Some details here:
http://geoinformaticstutorial.blogspot.no/2014/02/reading-binary-data-nsidc-sea-ice.html
Info on the ice concentration charts: http://nsidc.org/data/docs/daac/nsidc0051_gsfc_seaice.gd.html
Info on the map projection: http://nsidc.org/data/polar_stereo/ps_grids.html
The GeoTiff files are map projected to EPSG:3411, being the NSIDC-specific projection.
There also is produced a GeoTiff reprojected to EPSG:3575 which is the NP-standard for Barents/Fram-Strait.
Details on how to map project are found here:
http://geoinformaticstutorial.blogspot.no/2014/03/geocoding-nsidc-sea-ice-concentration.html
'''
#Define file names
(infilepath,
infilename) = os.path.split(infile) #get path and filename seperately
(infileshortname, extension) = os.path.splitext(infilename)
outfile = outfilepath + infileshortname + '.tif'
#Dimensions from https://nsidc.org/data/docs/daac/nsidc0051_gsfc_seaice.gd.html
height = 448
width = 304
#####
# READ FLAT BINARY INTO ARRAY
#####
#for this code on how to read flat binary string, inspiration found at https://stevendkay.wordpress.com/category/python/
icefile = open(infile, "rb")
contents = icefile.read()
icefile.close()
# unpack binary data into a flat tuple z
#offset and width/height from https://nsidc.org/data/docs/daac/nsidc0051_gsfc_seaice.gd.html
s = "%dB" % (int(width * height), )
z = struct.unpack_from(s, contents, offset=300)
nsidc = numpy.array(z).reshape((448, 304))
########
#WRITE THE ARRAY TO GEOTIFF
########
driver = gdal.GetDriverByName("GTiff")
outraster = driver.Create(outfile, width, height, 1, gdal.GDT_Int16)
if outraster is None:
print 'Could not create '
return
#set geotransform, values from https://nsidc.org/data/docs/daac/nsidc0051_gsfc_seaice.gd.html
geotransform = (-3850000.0, 25000.0, 0.0, 5850000.0, 0.0, -25000.0)
outraster.SetGeoTransform(geotransform)
outband = outraster.GetRasterBand(1)
#Write to file
outband.WriteArray(nsidc)
spatialRef = osr.SpatialReference()
#spatialRef.ImportFromEPSG(3411) --> this one does for some reason NOT work, but using proj4 does
spatialRef.ImportFromProj4(
'+proj=stere +lat_0=90 +lat_ts=70 +lon_0=-45 +k=1 +x_0=0 +y_0=0 +a=6378273 +b=6356889.449 +units=m +no_defs'
)
outraster.SetProjection(spatialRef.ExportToWkt())
outband.FlushCache()
#Clear arrays and close files
outband = None
outraster = None
nsidc = None
#####
#REPROJECT GEOTIFF TO EPSG3575
#####
EPSG3411_2_EPSG3575(outfile)
import gdal
input_file_name = "Data/pecs_tifs/pressure/09_03_10_50_CTP.tif"
output_file_name = "Data/pecs_tifs/pressure/09_03_10_50_CTP_scaled.tif"
tiff_file = gdal.Open(input_file_name)
# Store the GeoTiff data that we will insert back into the scaled file
geotransform = tiff_file.GetGeoTransform()
projection = tiff_file.GetProjection()
band = tiff_file.GetRasterBand(1)
xsize = band.XSize
ysize = band.YSize
# Get the data from the file
array = band.ReadAsArray()
tiff_file = None
band = None
# Manipulate that data however you'd like
array = array * 0.100000001490116
# Create a new tiff using the data we just manipulated and the old GeoTiff data we stored
driver = gdal.GetDriverByName('GTiff')
new_tiff = driver.Create(output_file_name, xsize, ysize, 1, gdal.GDT_Int16)
new_tiff.SetGeoTransform(geotransform)
new_tiff.SetProjection(projection)
new_tiff.GetRasterBand(1).WriteArray(array)
new_tiff.FlushCache()
new_tiff = None
def main(chemfile, dicamfile, maskfile, hyperfile):
########################################################
# Open data
chem = gdal.Open(chemfile)
dicam = gdal.Open(dicamfile)
mask = gdal.Open(maskfile)
geotrans = dicam.GetGeoTransform()
chemmeta = chem.GetMetadata('ENVI')
xoff = 0
yoff = 0
xcountchem = chem.RasterXSize
ycountchem = chem.RasterYSize
xcountmask = mask.RasterXSize
ycountmask = mask.RasterYSize
xcountdicam = dicam.RasterXSize
ycountdicam = dicam.RasterYSize
# Read raster as arrays
chemdata = chem.ReadAsArray(xoff, yoff, xcountchem, ycountchem).astype(np.float)
chemdata = chemdata.reshape((1, ycountchem, xcountchem))
dicamdata = dicam.ReadAsArray(xoff, yoff, xcountdicam, ycountdicam).astype(np.float)
maskdata = mask.ReadAsArray(xoff, yoff, xcountmask, ycountmask).astype(np.float)
# average dicam data for the RGB bands down to chem spatial resolution
dicamcoarse = np.zeros((3, ycountchem, xcountchem), dtype=np.float)
## red = 45
## green = 27
## blue = 9
ratio = np.double(10)
good = np.logical_and(np.equal(maskdata, 1), np.isfinite(chemdata).reshape(maskdata.shape))
numgood = good.sum()
print "Number of Good pixels: %d" % (numgood)
roworig, colorig = np.indices(good.shape)
row = roworig[good]
col = colorig[good]
tempdata = chemdata[:, row, col]
histo, edges = np.histogram(tempdata, normed=True)
peak = np.argmax(histo)
for j in range(ycountchem):
for k in range(xcountchem):
## first, find where the shaded data are to avoid
tempdatared = dicamdata[0,np.floor(j*ratio).astype(int):np.floor(j*ratio+ratio).astype(int),\
np.floor(k*ratio).astype(int):np.floor(k*ratio+ratio).astype(int)]
tempdatagreen = dicamdata[1,np.floor(j*ratio).astype(int):np.floor(j*ratio+ratio).astype(int),\
np.floor(k*ratio).astype(int):np.floor(k*ratio+ratio).astype(int)]
tempdatablue = dicamdata[2,np.floor(j*ratio).astype(int):np.floor(j*ratio+ratio).astype(int),\
np.floor(k*ratio).astype(int):np.floor(k*ratio+ratio).astype(int)]
dicamcoarse[0,j,k] = np.mean(tempdatared)
dicamcoarse[1,j,k] = np.mean(tempdatagreen)
dicamcoarse[2,j,k] = np.mean(tempdatablue)
del tempdatared, tempdatagreen, tempdatablue
chemdata = chemdata[:, row, col]
dicamcoarse = dicamcoarse[:, row, col]
print "Reshaped arrays"
print "DiCAM: %d %d" % dicamcoarse.shape
print "VSWIR: %d %d" % chemdata.shape
GGt = np.linalg.inv(np.matmul(dicamcoarse, dicamcoarse.T))
B = np.matmul(np.matmul(chemdata, dicamcoarse.T), GGt)
randnum = ("%06d") % np.random.randint(0,high=100000)
np.save("fusion_matrix_hold_GGt_"+randnum+".npy", GGt)
np.save("fusion_matrix_hold_B_"+randnum+".npy", B)
print "Saved matrices to file"
print "Did matrix multiplication"
## create a "checkerboard" mask so that the high resolution data can be ordered in rows
## to match the low resolution data.
checkers = np.arange(1, (xcountchem*ycountchem)+1).reshape(ycountchem, xcountchem)
checkers100 = np.kron(checkers, np.ones((10,10))).astype(np.int64)
goodbig = np.kron(good, np.ones((10,10))).astype(bool)
checkers100good = checkers100[goodbig]
sortindex = np.argsort(checkers100good)
rowbig, colbig = np.indices(goodbig.shape)
rowbig = rowbig[goodbig].flatten()
colbig = colbig[goodbig].flatten()
rowbig = rowbig[sortindex]
colbig = colbig[sortindex]
np.save("fusion_matrix_hold_rowbig_"+randnum+".npy", rowbig)
np.save("fusion_matrix_hold_colbig_"+randnum+".npy", colbig)
print "Created, sorted and saved Big indices"
## Clear some memory
del chemdata, dicamcoarse
del rowbig, colbig, GGt
## load data in memap mode to conserve memory
rowbig = np.load("fusion_matrix_hold_rowbig_"+randnum+".npy", mmap_mode='r')
colbig = np.load("fusion_matrix_hold_colbig_"+randnum+".npy", mmap_mode='r')
## GGt = np.load("fusion_matrix_hold_GGt_"+randnum+".npy", mmap_mode='r')
## B = np.load("fusion_matrix_hold_B_"+randnum+".npy", mmap_mode='r')
print "Reloaded data"
numstuff = rowbig.shape[0]
numchunks = 10
checksizes = np.zeros((10), dtype=np.int64)
checksizes[:] = np.floor(numstuff/10.)
checksizes[-1] += numstuff % 10
starts = np.roll(checksizes, 1)
starts[0] = 0
starts = np.cumsum(starts)
ends = np.cumsum(checksizes)
## hypercam = np.zeros((52,ycountdicam,xcountdicam), dtype=np.float)
tempfile = path.join(mkdtemp(dir='/export/tmp'), randnum+'.dat')
hypercam = np.memmap(tempfile, dtype='float32', mode='w+', shape=(1,ycountdicam,xcountdicam))
for nums in np.arange(numchunks):
rowhold = rowbig[starts[nums]:ends[nums]]
colhold = colbig[starts[nums]:ends[nums]]
hypercam[:, rowhold, colhold] = np.matmul(B, dicamdata[:, rowhold, colhold])
print "Finished chunk %d of %d" % (nums, numchunks)
print "Filled Hypercam array"
# Create for target raster the same projection as for the value raster
driver = gdal.GetDriverByName('ENVI')
target_ds = driver.Create(hyperfile, xcountdicam, ycountdicam, chem.RasterCount, gdal.GDT_Float32, ["INTERLEAVE=BIL"])
raster_srs = osr.SpatialReference()
raster_srs.ImportFromWkt(chem.GetProjectionRef())
target_ds.SetProjection(raster_srs.ExportToWkt())
target_ds.SetGeoTransform(geotrans)
## target_ds.SetMetadataItem('wavelength_units', vswirmeta['wavelength_units'], 'ENVI')
## target_ds.SetMetadataItem('wavelength', vswirmeta['wavelength'], 'ENVI')
target_ds.GetRasterBand(1).WriteArray(hypercam[0,:,:])
del target_ds
del chem, dicam, hypercam
## remove temporary directory and file
try:
shutil.rmtree(os.path.dirname(tempfile))
except:
pass
def main(inimagefile, inshape, nodata):
""" apply_shape_mask.py
This is the main function. It takes an input Shapefile and an output image file
and applies the polygonms in the shapefile as a mask
image file.
"""
root = os.path.basename(inimagefile)[0:15]
## newroot = root.replace('-','_')
newroot = root
temp = os.path.dirname(inimagefile)
parts = temp.split(os.path.sep)
week = parts[-1]
inshapefull = '/data/gdcsdata/Research/Researcher/Knapp/Moorea_Weekly/Baseline_Cloud_masks/' + inshape
inshapefile = glob.glob(inshapefull)
if (len(inshapefile) == 0):
print("Cannot find this Shapefile for this image in:\n %s\n Quitting" %
(inshapefull))
sys.exit(0)
inshapefile = inshapefile[0]
outmaskedimage = os.path.splitext(inimagefile)[0] + "_masked.tif"
## Open the input image file
imgDS = gdal.Open(inimagefile, gdal.GA_ReadOnly)
if imgDS is None:
print("Error: Canonot open file %s" % (inimagefile))
sys.exit(1)
## Get the projection of the image
rasproj = imgDS.GetProjectionRef()
## Get the GeoTransform of the image
## the GeoTransform provides the upper left corner and pixel resolution
## information of the image
gt = imgDS.GetGeoTransform()
rxmin = gt[0]
rxmax = gt[0] + (imgDS.RasterXSize * gt[1])
rymin = gt[3] + (imgDS.RasterYSize * gt[5])
rymax = gt[3]
## put the image projection information into a Spatial Reference
## in order to get the EPSG projection code.
myosr = osr.SpatialReference()
myosr.ImportFromWkt(rasproj)
## get the raster projection's EPSG code
rasepsg = int(myosr.GetAttrValue("AUTHORITY", 1))
## Get the Shape driver from ogr.
drvshp = ogr.GetDriverByName('ESRI Shapefile')
## drvkml = ogr.GetDriverByName('KML')
## Open the SHapefile with the Shapefile driver
vecDS = drvshp.Open(inshapefile, 0) ## 0 means read only, 1 mean writable
if vecDS is None:
print("Error: Cannot open file %s" % (inshapefile))
imgDS = None
sys.exit(1)
## get the layer in the Shapefile. Usually, there is noyl one layer in a shapefile
layer = vecDS.GetLayer()
## get the spatial reference of the vector Shapefile and get its EPSG code
vecsrs = layer.GetSpatialRef()
vecepsg = int(vecsrs.GetAttrValue("AUTHORITY", 1))
## if the EPSG codes are not the same, then the projections are not the same.
## Advise the user of this and exit.
if (rasepsg != vecepsg):
print("Projection of Vector file is not the same as raster file")
print(
"Please use ogr2ogr to create a version of the Shapefile in the ")
print("same projection as the Image file")
imgDS, vecDS = None, None
print("Raster: %d Shape: %d" % (rasepsg, vecepsg))
sys.exit(1)
## There should only be one feature with the geometry of the area to cover.
## Get the feature count and the feature.
featcnt = layer.GetFeatureCount()
# Rasterize zone polygon to a raster
# Create memory target raster
## targetDS = gdal.GetDriverByName('MEM').Create('', imgDS.RasterXSize, imgDS.RasterYSize, 1, gdal.GDT_Byte)
targetDS = gdal.GetDriverByName('MEM').Create('', imgDS.RasterXSize,
imgDS.RasterYSize, 1,
gdal.GDT_Byte)
targetDS.SetGeoTransform(gt)
# Create for target raster the same projection as for the value raster
raster_srs = osr.SpatialReference()
raster_srs.ImportFromWkt(imgDS.GetProjectionRef())
targetDS.SetProjection(raster_srs.ExportToWkt())
# Rasterize polygon to raster
gdal.RasterizeLayer(targetDS, [1], layer, burn_values=[1])
## Get the rasterized polygon data and count the number of pixels.
## Because we set the value in the raster to 1, we should be able to simply sum them up.
polytemp = targetDS.GetRasterBand(1).ReadAsArray()
mask = np.greater(polytemp, 0)
outDS = gdal.GetDriverByName('GTiff').Create(
outmaskedimage,
imgDS.RasterXSize,
imgDS.RasterYSize,
imgDS.RasterCount,
imgDS.GetRasterBand(1).DataType,
options=['COMPRESS=LZW'])
outDS.SetGeoTransform(gt)
# Create for target raster the same projection as for the value raster
outDS.SetProjection(raster_srs.ExportToWkt())
for band in range(imgDS.RasterCount):
bandarr = imgDS.GetRasterBand(band + 1).ReadAsArray()
bandarr[mask] = nodata
outDS.GetRasterBand(band + 1).WriteArray(bandarr)
outDS.GetRasterBand(band + 1).SetNoDataValue(-9999)
outDS.FlushCache()
## close all 3 data sets by setting them to None
imgDS, vecDS, targetDS, outDS = None, None, None, None
in_ds = gdal.Open("/data/110515YN1.tiff")
print("open tif file succeed")
cols = in_ds.RasterXSize
rows = in_ds.RasterYSize
bands = in_ds.RasterCount
# image = in_band1.ReadAsArray() #<class 'tuple'>: (88576, 194304)
# shape = (88576, 194304)
# Pre = np.zeros((rows,cols,bands)).astype(np.uint8)
target_size = 10000 # target_size = 1024
# 获取Tif的驱动，为创建切出来的图文件做准备
gtif_driver = gdal.GetDriverByName("GTiff")
# 创建切出来的要存的文件（3代表3个波段，最后一个参数为数据类型，跟原文件一致）
out_ds = gtif_driver.Create('Pre.tif', cols, rows, 3)
print("create new tif file succeed")
# out_band1 = out_ds.GetRasterBand(1)
# out_band2 = out_ds.GetRasterBand(2)
# out_band3 = out_ds.GetRasterBand(3)
xBlockSize = target_size
yBlockSize = target_size
# for i in range(0, rows, yBlockSize):
# if i + yBlockSize < rows:
# numRows = yBlockSize
def merging_save(self):
print('start stacking')
format = "GTiff"
driver = gdal.GetDriverByName(format)
metadata = driver.GetMetadata()
outputname, _ = QFileDialog.getSaveFileName(self, 'Save file')
if outputname is None:
QMessageBox.question(self, 'Выберите снова', "ВЫберите снова файл",
QMessageBox.Ok, QMessageBox.Ok)
output = driver.Create(str(outputname), self.x, self.y, self.bands,
gdal.GDT_UInt16)
if self.landsat8.isChecked():
format = "GTiff"
allbands = numpy.stack(
(self.coastal_aerosol, self.blue, self.green, self.red,
self.NIR, self.SWIR1, self.SWIR2))
for i in range(self.bands):
output.GetRasterBand(i + 1).WriteArray(allbands[i])
elif self.landsat7.isChecked() or self.landsat5.isChecked():
allbands = numpy.stack((self.blue, self.green, self.red, self.NIR,
self.SWIR1, self.TIR, self.SWIR2))
for i in range(self.bands):
output.GetRasterBand(i + 1).WriteArray(allbands[i])
elif self.sentinel2.isChecked():
blue = output.GetRasterBand(1)
blue.SetDescription('Blue band (2) Res 10m Wave 458-523')
blue.WriteArray(self.blue)
green = output.GetRasterBand(2)
green.SetDescription('Green band (3) Res 10m Wave 543-578')
green.WriteArray(self.green)
red = output.GetRasterBand(3)
red.SetDescription('Red band (4) Res 10m Wave 650-680')
red.WriteArray(self.red)
VRE1 = output.GetRasterBand(4)
VRE1.SetDescription('VRE1 band (5) Res 20m Wave 698-713')
VRE1_scaled = ndimage.zoom(self.VRE1, 2, order=1)
VRE1.WriteArray(VRE1_scaled)
VRE2 = output.GetRasterBand(4)
VRE2.SetDescription('VRE2 band (6) Res 20m Wave 733-748')
VRE2_scaled = ndimage.zoom(self.VRE2, 2, order=1)
VRE2.WriteArray(VRE2_scaled)
VRE3 = output.GetRasterBand(6)
VRE3.SetDescription('VRE3 band (7) Res 20m Wave 773-793')
VRE3_scaled = ndimage.zoom(self.VRE3, 2, order=1)
VRE3.WriteArray(VRE3_scaled)
NIR = output.GetRasterBand(7)
NIR.SetDescription('NIR band (8) Res 10m Wave 785-899')
NIR.WriteArray(self.NIR)
SWIR1 = output.GetRasterBand(8)
SWIR1.SetDescription('SWIR1 band (11) Res 20m Wave 1565-1655')
SWIR1_scaled = ndimage.zoom(self.SWIR1, 2, order=1)
SWIR1.WriteArray(SWIR1_scaled)
SWIR2 = output.GetRasterBand(9)
SWIR2.SetDescription('SWIR2 band (12) Res 20m Wave 2100-2280')
SWIR2_scaled = ndimage.zoom(SWIR2, 2, order=1)
SWIR2.WriteArray(SWIR2_scaled)
output.SetProjection(self.proj)
output.SetGeoTransform(self.transform)
output = None
QMessageBox.question(
self, 'Сохранение завершено',
'Теперь вы можете открыть сохраненный файл в QGIS/ArcGIS',
QMessageBox.Ok, QMessageBox.Ok)
def extract_bbox(self,
bbox,
directory,
verbose = True,
):
"""Extracts NASS CDL data from the source file for the bounding box."""
# get the extents
xmin, ymin, xmax, ymax = bbox
# somwhat hack way to figure out the UTM
UTM = int(numpy.floor((0.5 * xmin + 0.5 * xmax + 180) / 6) + 1)
# extract the values for each year
for year in self.years:
s = self.statecodes[self.statenames[self.state]]
its = self.destination, year, s
decompressed = '{}/CDL_{}_{}.tif'.format(*its)
output = '{}/{}landuse.tif'.format(directory, year)
if os.path.isfile(output):
print('land use file {} exists'.format(output))
elif not os.path.isfile(decompressed):
print('warning: source file ' +
'{} does not exist'.format(decompressed))
else:
# get the values of the raster and the origin
values, corner = get_raster_table(decompressed, bbox, 'uint8')
# get the source, source reference, and the source band
source = gdal.Open(decompressed)
source_band = source.GetRasterBand(1)
# set the transform to the new origin
transform = source.GetGeoTransform()
transform = (corner[0], transform[1], transform[2], corner[1],
transform[4], transform[1])
# get a driver and set the projection and georeference
driver = gdal.GetDriverByName('GTiff')
destination = driver.Create(output,
len(values[0]),
len(values),
1,
gdal.GDT_Byte)
destination.SetGeoTransform(transform)
destination.SetProjection(source.GetProjection())
# set the metadata and get the destination band
destination.SetMetadata(source.GetMetadata())
destination_band = destination.GetRasterBand(1)
# copy the pertinent attributes to the band
destination_band.WriteArray(values, 0, 0)
ct = source_band.GetColorTable().Clone()
destination_band.SetColorTable(ct)
# transform the projection from WGS 1984 to NAD 1983
# (needs to be done)
# close up the files
source = None
destination = None
if verbose:
print('successfully extracted cropland data to new file')
if verbose: print('')
def kmeans(bands, n_clusters=8, max_iter=10, outname=None):
"""Perform KMeans clustering on input dataset.
http://scikit-learn.org/stable/modules/generated/sklearn.cluster.MiniBatchKMeans.html
http://scikit-learn.org/stable/modules/generated/sklearn.cluster.KMeans.html#sklearn.cluster.KMeans
Args:
bands (GDALDataset): either a list of datasets,
or a dataset with multiple bands
outname (str): The string output name and path
Returns:
GDALDataset: The opened output dataset
"""
if outname is None:
if isinstance(bands, list):
band_filename = get_band_filename(bands[0].GetFileList())
else:
band_filename = get_band_filename(bands.GetFileList())
band_path, band_filename = os.path.split(band_filename)
outname = os.path.join(band_path, '%s_kmeans.TIF' % band_filename.split('_')[0])
if os.path.exists(outname):
os.remove(outname)
# Define the classifier
clf = MiniBatchKMeans(
n_clusters=n_clusters,
max_iter=max_iter,
batch_size=10000,
max_no_improvement=100,
init_size=2000,
n_init=10, # default was 3
reassignment_ratio=0.05
)
# Read data from each band
test_data = []
if isinstance(bands, list):
shape = np.ma.shape(bands[0].GetRasterBand(1).ReadAsArray())
x_size = bands[0].RasterXSize
y_size = bands[0].RasterYSize
out_type = bands[0].GetRasterBand(1).DataType
geo_trans = bands[0].GetGeoTransform()
geo_proj = bands[0].GetProjectionRef()
meta_data = bands[0].GetMetadata()
for b in bands:
b_array = b.GetRasterBand(1).ReadAsArray()
test_data.append(b_array.flatten())
else:
shape = np.ma.shape(bands.GetRasterBand(1).ReadAsArray())
x_size = bands.RasterXSize
y_size = bands.RasterYSize
out_type = bands.GetRasterBand(1).DataType
geo_trans = bands.GetGeoTransform()
geo_proj = bands.GetProjectionRef()
meta_data = bands.GetMetadata()
for band in range(bands.RasterCount):
b_array = bands.GetRasterBand(band+1).ReadAsArray()
shape = np.ma.shape(b_array)
test_data.append(b_array.flatten())
# Convert to float to prevent sklearn error/warning message
test_data = np.array(test_data, dtype=np.float32)
test_data = np.transpose(test_data)
# Performing K-means classification
clf.fit(test_data)
predictedClass = clf.predict(test_data)
predictedClass = predictedClass + 1 #Add 1 to exclude zeros in output raster
predictedClass = np.reshape(predictedClass, shape) # Reshape the numpy array to match the original image
# Create an output raster the same size as the input image
drv = gdal.GetDriverByName('GTiff')
fout = drv.Create(
outname,
x_size,
y_size,
1,
out_type
)
fout.SetGeoTransform(geo_trans)
fout.SetProjection(geo_proj)
fout.SetMetadata(meta_data)
# Write classification to band 1
fout.GetRasterBand(1).WriteArray(predictedClass)
fout = None
return gdal.Open(outname)
def create_geotiffs(file_dir, geotiff_dir):
#Specify the relative file location
start_date = '01/01/2002' #Date that GRACE data is available from
for file in os.listdir(file_dir): #Looping through the directory
if file is None:
print "No files to parse"
sys.exit()
if file.endswith('.nc'):
nc_fid = Dataset(file_dir + file, 'r') #Reading the netcdf file
nc_var = nc_fid.variables #Get the netCDF variables
nc_var.keys() #Getting variable keys
time = nc_var[
'time'][:] #Get the all the avaialable timesteps. Timestep increment value is x days after startdate
lwe_thickness = nc_var[
'lwe_thickness'][:, :, :] #Array with the all the values for lwe_thickness
date_str = datetime.strptime(start_date,
"%m/%d/%Y") #Start Date string.
var = "lwe_thickness" #Specifying the variable key. This parameter will be used to retrieve information about the netCDF file
xsize, ysize, GeoT, Projection, NDV = get_netcdf_info(
file_dir + file, var) #Get information about the netCDF file
#Looping through each timestep
for timestep, v in enumerate(time):
current_time_step = nc_var['lwe_thickness'][
timestep, :, :] #Getting the index of the current timestep
end_date = date_str + timedelta(
days=float(v)) #Actual human readable date of the timestep
ts_file_name = end_date.strftime(
"%Y_%m_%d") #Changing the date string format
fts_vals = set(
) #Creating an empty set to store the values for the timestep
for i in current_time_step.compressed(
): #Compressed ignores null values and returns an array of values that exist
if float(
i
) not in fts_vals: #Check if the value exists in the fts_values. If not then add the values.
fts_vals.add(float(i))
x = [
] #Creating an empty list to store the x indexes for a given value
y = [
] #Creating an empty list to store the y indexes for a given value
for i in fts_vals: #Looping through the existing values to get the indexes of x,y
idx = np.where(
nc_var['lwe_thickness'][timestep, :, :] == float(
i)) #Find the index of the given value
x = x + idx[0].tolist(
) #Write the x index values to the empty x list
y = y + idx[1].tolist(
) #Write the y index values to the empty y list
x_y = zip(x, y) #Combining the x,y list
grace_points = [
] #Creating an empty list to store a list of json dictionaries. Will be used to generate the shapefile.
for i in x_y: #Looping through the indexes to find the exact latitude and longitude of an existing value
grace_json = {
} #Empty json object to store the corresponding latitude, longitude and lwe thickness value
latitude = nc_var['lat'][i[0]]
longitude = nc_var['lon'][i[1]]
thickness = nc_var['lwe_thickness'][timestep, i[0], i[1]]
#Saving all the values to the jon dictionary
grace_json["latitude"] = latitude
grace_json["longitude"] = longitude
grace_json["thickness"] = thickness
grace_points.append(grace_json)
#Creating the shapefile from the json dictionaries, then converting it to a raster
try:
file_name = 'grace_sites'
temp_dir = tempfile.mkdtemp(
) #Creating a temporary directory to save the shapefile
file_location = temp_dir + "/" + file_name
w = sf.Writer(sf.POINT) #Creating a point shapefile
w.field(
'thickness'
) #Creating an attribute field called thickness for storing the variable value
#Looping through the list of json dictionaries to create points
for item in grace_points:
w.point(float(item['longitude']),
float(item['latitude'])) #Creating the point
w.record(item['thickness'],
'Point') #Assigning value to the point
w.save(file_location)
#Creating a projection file for the shapefile
prj = open("%s.prj" % file_location, "w")
epsg = 'GEOGCS["WGS84",DATUM["WGS_1984",SPHEROID["WGS84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.01745329251994328,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]'
prj.write(epsg)
prj.close()
#Begin the conveersion to a raster
NoData_value = -9999 #Specifying no data value
shp_file = file_location + ".shp" #Find the shapefile location
out_loc = geotiff_dir + ts_file_name + ".tif" #Specify the GeoTiff name and output
source_ds = ogr.Open(shp_file) #Reading the shapefile
source_layer = source_ds.GetLayer(
) #Getting the actual layer
spatialRef = source_layer.GetSpatialRef(
) #Get the Spatial Reference
raster_layer = gdal.GetDriverByName('GTiff').Create(
out_loc, xsize, ysize, 1,
gdal.GDT_Float32) #Initializing an empty GeoTiff
raster_layer.SetProjection(spatialRef.ExportToWkt(
)) #Set the projection based on the shapefile projection
raster_layer.SetGeoTransform(GeoT) #Set the Geotransform.
band = raster_layer.GetRasterBand(
1) #Specifying the number of bands
band.SetNoDataValue(NoData_value) #Setting no data values
band.FlushCache(
) #This call will recover memory used to cache data blocks for this raster band, and ensure that new requests are referred to the underlying driver.
gdal.RasterizeLayer(raster_layer, [1],
source_layer,
options=["ATTRIBUTE=thickness"
]) #Create the GeoTiff layer
except:
print "Error parsing the data. Please check directory and try again."
sys.exit()
return False
finally:
# Delete the temp shapefile dir after uploading the shapefile
if temp_dir is not None:
if os.path.exists(temp_dir):
shutil.rmtree(temp_dir)
def gmm_cluster():
data_path = "C:\\Users\\runaas.NORKART\\PycharmProjects\\SentinelTest\\data"
img_dir_path = os.path.join(
data_path,
"S2B_MSIL2A_20180821T104019_N0208_R008_T32VNM_20180821T170337\\S2B_MSIL2A_20180821T104019_N0208_R008_T32VNM_20180821T170337.SAFE\\GRANULE\\L2A_T32VNM_A007613_20180821T104015\\IMG_DATA\\R10m"
)
image_names = [
"T32VNM_20180821T104019_B02_10m.jp2",
"T32VNM_20180821T104019_B03_10m.jp2",
"T32VNM_20180821T104019_B04_10m.jp2",
"T32VNM_20180821T104019_B08_10m.jp2"
]
np_bands, cols, rows, xform, proj = training_data.image_set_load(
img_dir_path, image_names)
X = np.concatenate(
[b.reshape((b.shape[0] * b.shape[1], 1)) for b in np_bands], axis=1)
cut = X[[1, 3, 4], :]
title = "Diriclet prior"
gmm = mix.BayesianGaussianMixture(n_components=10,
reg_covar=0,
init_params='random',
max_iter=1500,
mean_precision_prior=.8)
gmm.fit(X[np.random.choice(X.shape[0], 50000), :])
prediction = gmm.predict(X)
prediction = prediction.reshape(np_bands[0].shape)
prediction = np.array(prediction, dtype=np.ubyte)
# Save result
tiff_driver = gdal.GetDriverByName('GTiff')
out_fn = os.path.join(img_dir_path, "clustered.tif")
outRaster = tiff_driver.Create(out_fn, prediction.shape[0],
prediction.shape[1], 1, gdal.GDT_Byte)
outRaster.SetGeoTransform(xform)
outRaster.SetProjection(proj)
outRaster.GetRasterBand(1).WriteArray(prediction)
outRaster.FlushCache()
plt.figure(figsize=(4.7 * 3, 8))
plt.subplots_adjust(bottom=.04,
top=0.90,
hspace=.05,
wspace=.05,
left=.03,
right=.99)
gs = gridspec.GridSpec(2, 3)
for k in range(3):
plot_results(plt.subplot(gs[0:2, k]),
X[:, k:k + 2],
gmm,
r"%s$%d$%d" % (title, k, k + 1),
plot_title=(k == 0))
plt.show()
