def interpolate(self, interpdf, output_path, generate_plots= True, format= 'csv'):
"""
Function that makes Kringe interpolation on the preprocessed wl data.
Inputs:
format: 'csv' : generates csv files in folder output_path/csv_files
'tif': generates tif files in folder output_path/tif_files
"""
# Get raster of Wadden Sea
extent, dims, crs, ws = utils.get_raster_wadden_sea()
# grid to make interpolation on
x= np.linspace(extent[0], extent[2], dims[0])
y = np.linspace(extent[1],extent[3], dims[1])
# For each date, make an interpolation using data, on grid x,y
#for row in range(interpdf.shape[0]):
for row in [0]:
print('Interpolating data at index %d ...'%row)
date= interpdf.loc[row,'date']
no_obs = int(len(interpdf.iloc[row, 1:].dropna().values)/3.)
data = interpdf.iloc[row, 1:].dropna().values.reshape(no_obs, 3)
# Do the linear interpolation
UK = OrdinaryKriging(data[:, 0], data[:, 1], data[:, 2],
variogram_model='power', coordinates_type= 'euclidean',
verbose= False, enable_plotting= False)
# kriged points and the variance.
z, ss = UK.execute('grid',x,y)
watlev = z.data*ws
watlev1 = np.where(watlev !=0, watlev, np.nan)
if (format == 'tif'):
#Convert array to raster.
opath= os.path.join(output_path,'tif_files/')
if not os.path.exists(opath):
os.makedirs(opath)
utils.array2raster(463, extent[0], extent[3], crs, watlev, opath, raster_name= 'wl_%d-%02d-%02d.tif'%(date.year,date.month, date.day) )
if (format== 'csv'):
opath= os.path.join(output_path,'csv_files/')
if not os.path.exists(opath):
os.makedirs(opath)
table= utils.arraytocsv(watlev1, dims[0], dims[1])
table.rename(columns={'value': 'water_level'}, inplace= True)
table.to_csv(opath+'wl_%d-%02d-%02d.csv'%(date.year,date.month, date.day), index= False)
#Create plot if true:
if (generate_plots):
path= os.path.join(output_path, 'plots/')
if not os.path.exists(path):
os.makedirs(path)
self.plotting(path, watlev1, interpdf.loc[row,'date'] , ofilename= '%05d.png'%row)
def save_images(fetches, step=None):
image_dir = os.path.join(a.output_dir, "images")
if not os.path.exists(image_dir):
os.makedirs(image_dir)
for i, in_path in enumerate(fetches["imnames"]):
name, _ = os.path.splitext(os.path.basename(in_path.decode("utf8")))
for kind in ["inputs1", "inputs2", "outputs", "targets"]:
filename = name + "-" + kind + ".tif"
if step is not None:
filename = "%08d-%s" % (step, filename)
out_path = os.path.join(image_dir, filename)
contents = fetches[kind][i]
if kind is "targets":
array2raster(out_path, [0, 0], 200, 200,
contents.transpose(2, 0, 1), 4)
elif kind is "outputs":
array2raster(out_path, [0, 0], 800, 800,
contents.reshape(800, 800, 4).transpose(2, 0, 1),
4)
elif kind is "inputs1":
array2raster(out_path, [0, 0], 800, 800,
contents.reshape(800, 800, 4).transpose(2, 0, 1),
4)
else:
array2raster(out_path, [0, 0], 800, 800,
contents.reshape(800, 800), 1)
def test_rast2array2rast(self):
ds = gdal.Open(self.template_fn)
array = utils.ds2array(ds)
self.assertEquals(array.shape, (45, 54))
rast_fn = utils.array2raster(array, self.template_fn)
self.assertEqual(rast_fn, '/tmp/output_dummy_single_band.tif')
os.remove(rast_fn)
def test_rast2array2rast(self):
ds = gdal.Open(self.template_fn)
array = utils.ds2array(ds)
self.assertEquals(array.shape, (45, 54))
rast_fn = utils.array2raster(array, self.template_fn)
self.assertEqual(rast_fn, '/tmp/output_dummy_single_band.tif')
os.remove(rast_fn)
def save_images(fetches, step=None):
image_dir = os.path.join(a.output_dir, "images")
if not os.path.exists(image_dir):
os.makedirs(image_dir)
for i in range((fetches["outputs_1"].shape[0])):
name = '%d' % i
for kind in ["outputs", "targets"]:
if a.mode == "train":
filename = "train-" + name + "-" + kind + ".tif"
if step is not None:
filename = "%08d-%s" % (step, filename)
else:
name = '%d' % (i + a.batch_size * step)
filename = "test-" + name + "-" + kind + ".tif"
out_path = os.path.join(image_dir, filename)
contents = fetches[kind][i]
if kind in ["outputs", "targets"]:
array2raster(out_path, [0, 0], 128, 128, contents.transpose(2, 0, 1), 4)
def test_array2raster_invalid_dim(self):
ds = gdal.Open(self.template_fn)
array = utils.ds2array(ds)
utils.array2raster(array.transpose(), self.template_fn)
}
}
test_cnt = {
32: {
"QB": 486,
"GF-2": 3965,
"QB_origin": 486,
"GF-2_origin": 3965
},
64: {
"QB": 117,
"GF-2": 960,
"QB_origin": 117,
"GF-2_origin": 960
}
}
cnt = train_cnt[blk][satellite] if mode == 'train' else test_cnt[blk][
satellite]
dataDir = '/data/zh/PSData/Dataset/%s/%s_%d' % (satellite, mode, blk)
rasterOrigin = (-123.25745, 45.43013)
for num in range(cnt):
dt_pan = gdal.Open('%s/%d_pan.tif' % (dataDir, num)).ReadAsArray()
dt_pan_d = cv2.resize(cv2.resize(dt_pan, (blk, blk)),
(blk * 4, blk * 4))
array2raster('%s/%d_pan_d.tif' % (dataDir, num), rasterOrigin, 2.4,
2.4, dt_pan_d, 1)
print("done")
def interpolate(self, generate_plots=True, format='csv'):
"""
Function that makes Kringe interpolation on the concatenated surge data.
Inputs:
format: 'csv' : generates csv files in folder output_path/csv_files
'tif': generates tif files in folder output_path/tif_files
"""
# Get the surge in format: date| x1|y1|wl1|...|xn|yn|wln|
surgedf = self.concat_surge_tables()
# Get raster of Wadden Sea
extent, dims, crs, ws = utils.get_raster_wadden_sea()
# grid to make interpolation on
x = np.linspace(extent[0], extent[2], dims[0])
y = np.linspace(extent[1], extent[3], dims[1])
# For each date, make an interpolation using data, on grid x,y
#for index, row in surgedf.iterrows():
# date= row['date']
#for row in [0]:
for row in range(surgedf.shape[0]):
print('Interpolating data at index %d ...' % row)
date = surgedf.loc[row, 'date']
no_obs = int(len(surgedf.iloc[row, 1:].dropna().values) / 3.)
data = surgedf.iloc[row, 1:].dropna().values.reshape(no_obs, 3)
# Do the linear interpolation
UK = OrdinaryKriging(data[:, 0],
data[:, 1],
data[:, 2],
variogram_model='power',
coordinates_type='euclidean',
verbose=False,
enable_plotting=False)
# kriged points and the variance.
z, ss = UK.execute('grid', x, y)
surge = z.data * ws # To get the surge in points inside Wadden and not in the whole bbox
surge1 = np.where(surge != 0, surge, np.nan)
if (format == 'tif'):
#Convert array to raster.
opath = os.path.join(self.opath, 'tif_files/')
if not os.path.exists(opath):
os.makedirs(opath)
utils.array2raster(463,
extent[0],
extent[3],
crs,
surge,
opath,
raster_name='surge_%d-%02d-%02d.tif' %
(date.year, date.month, date.day))
if (format == 'csv'):
opath = os.path.join(self.opath, 'csv_files/')
if not os.path.exists(opath):
os.makedirs(opath)
table = utils.arraytocsv(surge1, dims[0], dims[1])
table.rename(columns={'value': 'surge'}, inplace=True)
table.to_csv(opath + 'surge_%d-%02d-%02d.csv' %
(date.year, date.month, date.day),
index=False)
if (generate_plots):
path = os.path.join(self.opath, 'plots/')
if not os.path.exists(path):
os.makedirs(path)
self.plotting(path,
surge1,
surgedf.loc[row, 'date'],
ofilename='%05d.png' % row)
h, w = rawMul.shape[:2]
h = h // 4 * 4
w = w // 4 * 4
imgMul = cv2.resize(rawMul, (w, h))
imgLR = cv2.resize(imgMul, (w // 4, h // 4))
imgLR_U = cv2.resize(imgLR, (w, h))
imgPan = cv2.resize(rawPan, (w, h))
imgPan_D = upsample(downsample(imgPan))
imgMul = imgMul.transpose(2, 0, 1)
imgLR = imgLR.transpose(2, 0, 1)
imgLR_U = imgLR_U.transpose(2, 0, 1)
array2raster(newMul, rasterOrigin, 2.4, 2.4, imgMul, 4) # mul
array2raster(newLR_U, rasterOrigin, 2.4, 2.4, imgLR_U, 4) # lr_u
array2raster(newLR, rasterOrigin, 2.4, 2.4, imgLR, 4) # lr
array2raster(newPan, rasterOrigin, 2.4, 2.4, imgPan, 1) # pan
array2raster(newPan_D, rasterOrigin, 2.4, 2.4, imgPan_D, 1) # pan_d
print('mul:', imgMul.shape, ' lr_u:', imgLR_U.shape, ' lr:',
imgLR.shape, ' pan:', imgPan.shape, ' pan_d:', imgPan_D.shape)
print('done%s' % i)
# origin
# MUL(crop 1/4) -> mul_o(1), mul_o_u(1*4)
# PAN(crop 1/4) -> pan_o(1), pan_o_d(1/4*4)
for i in imagesIndex2:
newMul_o = '%s/Dataset/%s/%d_mul_o.tif' % (dataDir, satellite, i)
newMul_o_u = '%s/Dataset/%s/%d_mul_o_u.tif' % (dataDir, satellite, i)
def test_array2raster_invalid_dim(self):
ds = gdal.Open(self.template_fn)
array = utils.ds2array(ds)
utils.array2raster(array.transpose(), self.template_fn)
img_mul = dt_mul.ReadAsArray() # (c, h, w)
img_lr = dt_lr.ReadAsArray()
img_lr_u = dt_lr_u.ReadAsArray()
img_pan = dt_pan.ReadAsArray()
img_pan_d = dt_pan_d.ReadAsArray()
XSize = dt_lr.RasterXSize
YSize = dt_lr.RasterYSize
sample = int (XSize * YSize / blk / blk * ratio)
for _ in range(sample):
x = random.randint(0, XSize - blk)
y = random.randint(0, YSize - blk)
array2raster('%s/%d_mul.tif' % (trainDir, trainCount), rasterOrigin, 2.4, 2.4,
img_mul[:, y * 4:(y + blk) * 4, x * 4:(x + blk) * 4], 4)
array2raster('%s/%d_lr_u.tif' % (trainDir, trainCount), rasterOrigin, 2.4, 2.4,
img_lr_u[:, y * 4:(y + blk) * 4, x * 4:(x + blk) * 4], 4)
array2raster('%s/%d_lr.tif' % (trainDir, trainCount), rasterOrigin, 2.4, 2.4,
img_lr[:, y:(y + blk), x:(x + blk)], 4)
array2raster('%s/%d_pan.tif' % (trainDir, trainCount), rasterOrigin, 2.4, 2.4,
img_pan[y * 4:(y + blk) * 4, x * 4:(x + blk) * 4], 1)
array2raster('%s/%d_pan_d.tif' % (trainDir, trainCount), rasterOrigin, 2.4, 2.4,
img_pan_d[y * 4:(y + blk) * 4, x * 4:(x + blk) * 4], 1)
trainCount += 1
print ("done %d" % num)
record.write("%d\n" % trainCount)
record.close()
def interpolate(self, generate_plots=True, format='csv'):
"""
Interpolated the concatenated data.
"""
self.concatenate_clean_tables() # update salinity to desired structure
# Get of Wadden Sea
extent, dims, crs, ws = utils.get_raster_wadden_sea()
# grid to make interpolation on
x = np.linspace(extent[0], extent[2], dims[0])
y = np.linspace(extent[1], extent[3], dims[1])
# For each date, make an interpolation using data, on grid x,y
#for row in [0]:
for row in range(self.salinity.shape[0]):
print('Interpolating data at index %d ...' % row)
date = self.salinity.loc[row, 'time']
no_obs = int(len(self.salinity.iloc[row, 1:].dropna().values) / 3.)
data = self.salinity.iloc[row, 1:].dropna().values.reshape(
no_obs, 3) #x, y, salinity
# Do the linear interpolation
UK = OrdinaryKriging(data[:, 0],
data[:, 1],
data[:, 2],
variogram_model='power',
coordinates_type='euclidean',
verbose=False,
enable_plotting=False)
# kriged points and the variance.
z, ss = UK.execute('grid', x, y)
salinity = z.data * ws # on land, salinity is zero
salinity1 = np.where(salinity != 0, salinity, np.nan)
if (format == 'tif'):
#Convert array to raster.
opath = os.path.join(self.opath, 'tif_files/')
if not os.path.exists(opath):
os.makedirs(opath)
utils.array2raster(463,
extent[0],
extent[3],
crs,
salinity,
opath,
raster_name='salinity_%d-%02d-%02d.tif' %
(date.year, date.month, date.day))
if (format == 'csv'):
opath = os.path.join(self.opath, 'csv_files/')
if not os.path.exists(opath):
os.makedirs(opath)
table = utils.arraytocsv(salinity1, dims[0], dims[1])
table.rename(columns={'value': 'salinity'}, inplace=True)
table.to_csv(opath + 'salinity_%d-%02d-%02d.csv' %
(date.year, date.month, date.day),
index=False)
if (generate_plots):
path = os.path.join(self.opath, 'plots/')
if not os.path.exists(path):
os.makedirs(path)
self.plotting(path,
salinity1,
self.salinity.loc[row, 'time'],
ofilename='%05d.png' % row)