def test_geotiff_writer_create_invalid_number_of_data_types():
with raises(
ValueError,
message=
'List with data types must be of same size as list of file names'):
file_names = [
os.path.abspath('{}/name41.tif'.format(GEOTIFF_WRITE_FOLDER)),
os.path.abspath('{}/name42.tif'.format(GEOTIFF_WRITE_FOLDER)),
os.path.abspath('{}/name43.tif'.format(GEOTIFF_WRITE_FOLDER)),
os.path.abspath('{}/name44.tif'.format(GEOTIFF_WRITE_FOLDER)),
os.path.abspath('{}/name45.tif'.format(GEOTIFF_WRITE_FOLDER))
]
try:
num_bands = [1, 2, 3, 1, 1]
data_types = ['Float', 'Double', 'Int', 'Double']
writer = GeoTiffWriter(file_names,
GEO_TRANSFORM,
PROJECTION,
3,
4,
num_bands,
data_types=data_types)
writer.close()
finally:
for file_name in file_names:
if os.path.exists(file_name):
os.remove(file_name)
def create_kaska_s1_inference_output_files(s1_stack_file_dir: str,
priors_dir: str,
output_directory: str,
parameters: Optional[List[str]],
state_mask: Optional[str] = None,
roi: Optional[Union[str, Polygon]] = None,
spatial_resolution: Optional[int] = None,
roi_grid: Optional[str] = None,
destination_grid: Optional[str] = None):
mask_data_set, untiled_reprojection = _get_mask_data_set_and_reprojection(state_mask, spatial_resolution, roi,
roi_grid, destination_grid)
s1_stack_file = glob.glob(os.path.join(s1_stack_file_dir, 's1_nc_stack*.nc'))[0]
sar = get_sar(s1_stack_file)
time = [datetime(1970, 1, 1) + timedelta(days=float(i)) for i in sar.time]
s1_doys = np.array([i.timetuple().tm_yday for i in time])
lai_min_doy, lai_max_doy = _get_min_max_doy_from_s2_param('lai', priors_dir)
cab_min_doy, cab_max_doy = _get_min_max_doy_from_s2_param('cab', priors_dir)
cb_min_doy, cb_max_doy = _get_min_max_doy_from_s2_param('cb', priors_dir)
min_doy = min(lai_min_doy, cab_min_doy, cb_min_doy)
max_doy = max(lai_max_doy, cab_max_doy, cb_max_doy)
time_mask = (s1_doys >= min_doy) & (s1_doys <= max_doy)
times = [i.strftime('%Y%j') for i in np.array(time)[time_mask]]
outfile_names = []
for parameter_name in parameters:
for time_step in times:
outfile_names.append(os.path.join(output_directory, f'{parameter_name}_A{time_step}.tif'))
writer = GeoTiffWriter(outfile_names, mask_data_set.GetGeoTransform(), mask_data_set.GetProjection(),
mask_data_set.RasterXSize, mask_data_set.RasterYSize,
num_bands=None, data_types=None)
writer.close()
def create_kaska_s2_inference_output_files(start_time: Union[str, datetime],
end_time: Union[str, datetime],
time_step: Union[int, timedelta],
forward_models: List[str],
output_directory: str,
parameters: Optional[List[str]] = None,
state_mask: Optional[str] = None,
roi: Optional[Union[str, Polygon]] = None,
spatial_resolution: Optional[int] = None,
roi_grid: Optional[str] = None,
destination_grid: Optional[str] = None,
):
if type(start_time) is str:
start_time = get_time_from_string(start_time)
if type(end_time) is str:
end_time = get_time_from_string(end_time)
if type(time_step) is int:
time_step = timedelta(days=time_step)
time_grid = []
current_time = start_time
while current_time < end_time:
time_grid.append(current_time)
current_time += time_step
time_grid.append(end_time)
mask_data_set, untiled_reprojection = _get_mask_data_set_and_reprojection(state_mask, spatial_resolution, roi,
roi_grid, destination_grid)
model_parameter_names = []
other_logger.info('Assembling model parameter names')
for forward_model_name in forward_models:
forward_model = get_forward_model(forward_model_name)
if forward_model is None:
other_logger.warning(f'Could not find forward model {forward_model_name}')
continue
for variable in forward_model.variables:
other_logger.info(f'Checking variable {variable}')
if variable not in model_parameter_names:
model_parameter_names.append(variable)
outfile_names = []
requested_indexes = []
for i, parameter_name in enumerate(model_parameter_names):
other_logger.info(f'Checking for {parameter_name}')
if parameters is None or parameter_name in parameters:
other_logger.info(f'Creating output files for {parameter_name}')
requested_indexes.append(i)
for time_step in time_grid:
time = time_step.strftime('%Y%j')
outfile_names.append(f"{output_directory}/{parameter_name}_A{time}.tif")
other_logger.info(f'Created output file {parameter_name}')
writer = GeoTiffWriter(outfile_names, mask_data_set.GetGeoTransform(), mask_data_set.GetProjection(),
mask_data_set.RasterXSize, mask_data_set.RasterYSize, num_bands=None, data_types=None)
writer.close()
def __init__(self,
actual_parameter_list: List[str],
full_parameter_list: List[str],
output_folder: str,
time_step: datetime,
geo_transform: tuple,
projection: str,
width: int,
height: int,
output_format: str = "GeoTiff",
state_folder: str = None):
self._actual_parameter_list = actual_parameter_list
file_names = []
self.param_positions = {}
num_bands = []
for param in actual_parameter_list:
if param in full_parameter_list:
file_names.append(
os.path.join(
output_folder,
"%s_%s.tif" % (param, time_step.strftime("A%Y%j"))))
num_bands.append(1)
file_names.append(
os.path.join(
output_folder, "%s_%s_unc.tif" %
(param, time_step.strftime("A%Y%j"))))
num_bands.append(1)
self.param_positions[param] = full_parameter_list.index(param)
if output_format == "GeoTiff":
self.writer = GeoTiffWriter(file_names,
geo_transform,
projection,
width,
height,
num_bands,
data_types=None)
else:
raise ValueError(f"Format {output_format} is not supported.")
self._width = width
self._height = height
self._offset_x = 0
self._offset_y = 0
self.state_folder = state_folder
def test_geotiff_writer_create_invalid_data_types():
with raises(ValueError, message='Data Type dgfvbgf not supported.'):
file_names = [
os.path.abspath('{}/name31.tif'.format(GEOTIFF_WRITE_FOLDER)),
os.path.abspath('{}/name32.tif'.format(GEOTIFF_WRITE_FOLDER)),
os.path.abspath('{}/name33.tif'.format(GEOTIFF_WRITE_FOLDER)),
os.path.abspath('{}/name34.tif'.format(GEOTIFF_WRITE_FOLDER)),
os.path.abspath('{}/name35.tif'.format(GEOTIFF_WRITE_FOLDER))
]
try:
num_bands = [1, 2, 3, 1, 1]
data_types = ['Float', 'dgfvbgf', 'Double', 'Int', 'Double']
writer = GeoTiffWriter(file_names,
GEO_TRANSFORM,
PROJECTION,
3,
4,
num_bands,
data_types=data_types)
writer.close()
finally:
for file_name in file_names:
if os.path.exists(file_name):
os.remove(file_name)
def test_geotiff_writer_create():
file_names = [
os.path.abspath('{}/name11.tif'.format(GEOTIFF_WRITE_FOLDER)),
os.path.abspath('{}/name12.tif'.format(GEOTIFF_WRITE_FOLDER)),
os.path.abspath('{}/name13.tif'.format(GEOTIFF_WRITE_FOLDER)),
os.path.abspath('{}/name14.tif'.format(GEOTIFF_WRITE_FOLDER)),
os.path.abspath('{}/name15.tif'.format(GEOTIFF_WRITE_FOLDER))
]
try:
num_bands = [1, 2, 3, 1, 1]
data_types = ['Float', 'Float', 'Double', 'Int', 'Double']
writer = GeoTiffWriter(file_names,
GEO_TRANSFORM,
PROJECTION,
5,
5,
num_bands,
data_types=data_types)
writer.close()
assert writer is not None
finally:
for file_name in file_names:
if os.path.exists(file_name):
os.remove(file_name)
def infer_kaska_s1(s1_stack_file_dir: str,
priors_dir: str,
output_directory: str,
parameters: Optional[List[str]],
state_mask: Optional[str] = None,
roi: Optional[Union[str, Polygon]] = None,
spatial_resolution: Optional[int] = None,
roi_grid: Optional[str] = None,
destination_grid: Optional[str] = None,
tile_index_x: Optional[int] = 0,
tile_index_y: Optional[int] = 0,
tile_width: Optional[int] = None,
tile_height: Optional[int] = None
):
mask_data_set, untiled_reprojection = _get_mask_data_set_and_reprojection(state_mask, spatial_resolution, roi,
roi_grid, destination_grid)
reprojection = untiled_reprojection
tile_mask_data_set = mask_data_set
raster_width = mask_data_set.RasterXSize
raster_height = mask_data_set.RasterYSize
offset_x = 0
offset_y = 0
if tile_width is not None and tile_height is not None:
geo_transform = mask_data_set.GetGeoTransform()
ulx, xres, xskew, uly, yskew, yres = geo_transform
minlrx = ulx + (mask_data_set.RasterXSize * xres)
minlry = uly + (mask_data_set.RasterYSize * yres)
ulx = ulx + (tile_index_x * tile_width * xres)
uly = uly + (tile_index_y * tile_height * yres)
lrx = ulx + (tile_width * xres)
lry = uly + (tile_height * yres)
raster_width = tile_width
raster_height = tile_height
if (lrx > ulx and lrx > minlrx) or (lrx < ulx and lrx < minlrx):
lrx = minlrx
raster_width = int(np.abs((ulx - lrx) / xres))
if (lry > uly and lry > minlry) or (lry < uly and lry < minlry):
lry = minlry
raster_height = int(np.abs((uly - lry) / yres))
offset_x = tile_index_x * tile_width
offset_y = tile_index_y * tile_height
roi_bounds = (min(ulx, lrx), min(uly, lry), max(ulx, lrx), max(uly, lry))
destination_spatial_reference_system = osr.SpatialReference()
projection = mask_data_set.GetProjection()
destination_spatial_reference_system.ImportFromWkt(projection)
reprojection = Reprojection(roi_bounds, xres, yres, destination_spatial_reference_system)
tile_mask_data_set = reprojection.reproject(mask_data_set)
elif tile_width is not None or tile_height is not None:
logging.warning('To use tiling, parameters tileWidth and tileHeight must be set. Continue without tiling')
s1_stack_file = glob.glob(os.path.join(s1_stack_file_dir, 's1_nc_stack*.nc'))[0]
sar = get_sar(s1_stack_file)
s1_data = read_sar(sar, tile_mask_data_set)
s1_doys = np.array([i.timetuple().tm_yday for i in s1_data.time])
prior = _get_s1_priors(s1_doys, priors_dir, reprojection, raster_width, raster_height)
sar_inference_data = _get_sar_inference_data(s1_data, s1_doys, priors_dir, reprojection,
raster_width, raster_height)
lai_outputs, sr_outputs, sm_outputs, \
Avv_outputs, Bvv_outputs, Cvv_outputs, \
Avh_outputs, Bvh_outputs, Cvh_outputs, uorbits = do_inversion(sar_inference_data, prior, tile_mask_data_set, False)
times = [i.strftime('%Y-%m-%d') for i in np.array(sar_inference_data.time)[sar_inference_data.time_mask]]
outfile_names = []
for parameter_name in parameters:
for time_step in times:
outfile_names.append(os.path.join(output_directory, f's1_{parameter_name}_A{time_step}.tif'))
writer = GeoTiffWriter(outfile_names, mask_data_set.GetGeoTransform(), mask_data_set.GetProjection(),
mask_data_set.RasterXSize, mask_data_set.RasterYSize,
num_bands=None, data_types=None)
data = []
for parameter_name in parameters:
if parameter_name == 'lai':
for i in range(len(times)):
data.append(lai_outputs[i, :, :])
elif parameter_name == 'sr':
for i in range(len(times)):
data.append(sr_outputs[i, :, :])
elif parameter_name == 'sm':
for i in range(len(times)):
data.append(sr_outputs[i, :, :])
writer.write(data, raster_width, raster_height, offset_x, offset_y)
writer.close()
def infer_kaska_s2(start_time: Union[str, datetime],
end_time: Union[str, datetime],
time_step: Union[int, timedelta],
datasets_dir: str,
forward_models: List[str],
output_directory: str,
parameters: Optional[List[str]] = None,
state_mask: Optional[str] = None,
roi: Optional[Union[str, Polygon]] = None,
spatial_resolution: Optional[int] = None,
roi_grid: Optional[str] = None,
destination_grid: Optional[str] = None,
tile_index_x: Optional[int] = 0,
tile_index_y: Optional[int] = 0,
tile_width: Optional[int] = None,
tile_height: Optional[int] = None
):
if type(start_time) is str:
start_time = get_time_from_string(start_time)
if type(end_time) is str:
end_time = get_time_from_string(end_time)
if type(time_step) is int:
time_step = timedelta(days=time_step)
time_grid = []
current_time = start_time
while current_time < end_time:
time_grid.append(current_time)
current_time += time_step
time_grid.append(end_time)
if not os.path.exists(output_directory):
os.makedirs(output_directory)
temp_dir = f'{output_directory}/temp_{tile_index_x}_{tile_index_y}/'
if not os.path.exists(temp_dir):
os.makedirs(temp_dir)
mask_data_set, untiled_reprojection = _get_mask_data_set_and_reprojection(state_mask, spatial_resolution, roi,
roi_grid, destination_grid)
reprojection = untiled_reprojection
tile_mask_data_set = mask_data_set
raster_width = mask_data_set.RasterXSize
raster_height = mask_data_set.RasterYSize
offset_x = 0
offset_y = 0
if tile_width is not None and tile_height is not None:
geo_transform = mask_data_set.GetGeoTransform()
ulx, xres, xskew, uly, yskew, yres = geo_transform
minlrx = ulx + (mask_data_set.RasterXSize * xres)
minlry = uly + (mask_data_set.RasterYSize * yres)
ulx = ulx + (tile_index_x * tile_width * xres)
uly = uly + (tile_index_y * tile_height * yres)
lrx = ulx + (tile_width * xres)
lry = uly + (tile_height * yres)
raster_width = tile_width
raster_height = tile_height
if (lrx > ulx and lrx > minlrx) or (lrx < ulx and lrx < minlrx):
lrx = minlrx
raster_width = np.abs((ulx - lrx) / xres)
if (lry > uly and lry > minlry) or (lry < uly and lry < minlry):
lry = minlry
raster_height = np.abs((uly - lry) / yres)
offset_x = tile_index_x * tile_width
offset_y = tile_index_y * tile_height
roi_bounds = (min(ulx, lrx), min(uly, lry), max(ulx, lrx), max(uly, lry))
destination_spatial_reference_system = osr.SpatialReference()
projection = mask_data_set.GetProjection()
destination_spatial_reference_system.ImportFromWkt(projection)
reprojection = Reprojection(roi_bounds, xres, yres, destination_spatial_reference_system)
tile_mask_data_set = reprojection.reproject(mask_data_set)
elif tile_width is not None or tile_height is not None:
logging.warning('To use tiling, parameters tileWidth and tileHeight must be set. Continue without tiling')
file_refs = _get_valid_files(datasets_dir)
observations_factory = ObservationsFactory()
observations_factory.sort_file_ref_list(file_refs)
# an observations wrapper to be passed to kafka
observations = observations_factory.create_observations(file_refs, reprojection, forward_models)
model_parameter_names = []
other_logger.info('Assembling model parameter names')
for forward_model_name in forward_models:
forward_model = get_forward_model(forward_model_name)
if forward_model is None:
other_logger.warning(f'Could not find forward model {forward_model_name}')
continue
for variable in forward_model.variables:
other_logger.info(f'Checking variable {variable}')
if variable not in model_parameter_names:
model_parameter_names.append(variable)
outfile_names = []
requested_indexes = []
for i, parameter_name in enumerate(model_parameter_names):
other_logger.info(f'Checking for {parameter_name}')
if parameters is None or parameter_name in parameters:
other_logger.info(f'Creating output files for {parameter_name}')
requested_indexes.append(i)
for time_step in time_grid:
time = time_step.strftime('%Y-%m-%d')
outfile_names.append(f"{output_directory}/s2_{parameter_name}_A{time}.tif")
other_logger.info(f'Created output file {parameter_name}')
writer = GeoTiffWriter(outfile_names, mask_data_set.GetGeoTransform(), mask_data_set.GetProjection(),
mask_data_set.RasterXSize, mask_data_set.RasterYSize, num_bands=None, data_types=None)
data = []
at_least_one_valid_observation = False
for date in observations.dates:
granule = observations.read_granule(date)
if granule[0] is not None:
at_least_one_valid_observation = True
break
if not at_least_one_valid_observation:
logging.info('No valid observations found. Will skip inference.')
for j in requested_indexes:
for i in range(len(time_grid)):
data.append(np.zeros((int(raster_height), int(raster_width))))
else:
# todo make this more elaborate when more than one inverter is available
approx_inverter = get_inverter("prosail_5paras", "Sentinel2")
kaska = KaSKA(observations=observations,
time_grid=time_grid,
state_mask=tile_mask_data_set,
approx_inverter=approx_inverter,
output_folder=temp_dir,
save_sgl_inversion=False)
results = kaska.run_retrieval()
for j, sub_data in enumerate(results[1:]):
if j in requested_indexes:
for i in range(len(time_grid)):
data.append(sub_data[i, :, :])
other_logger.info(f'Writing to {offset_x}, {offset_y} with width {raster_width} and height {raster_height}')
writer.write(data, raster_width, raster_height, offset_x, offset_y)
def test_geotiff_writer_write():
file_names = [
os.path.abspath('{}/name51.tif'.format(GEOTIFF_WRITE_FOLDER)),
os.path.abspath('{}/name52.tif'.format(GEOTIFF_WRITE_FOLDER)),
os.path.abspath('{}/name53.tif'.format(GEOTIFF_WRITE_FOLDER)),
os.path.abspath('{}/name54.tif'.format(GEOTIFF_WRITE_FOLDER)),
os.path.abspath('{}/name55.tif'.format(GEOTIFF_WRITE_FOLDER))
]
try:
for file_name in file_names:
assert not os.path.exists(file_name)
num_bands = [1, 2, 3, 1, 1]
data_types = ['Float', 'Float', 'Double', 'Int', 'Double']
writer = GeoTiffWriter(file_names,
GEO_TRANSFORM,
PROJECTION,
3,
4,
num_bands,
data_types=data_types)
data_list = [
np.array([[0.70970884, 0.36247307, 0.66030969],
[0.93471179, 0.89365593, 0.63907102],
[0.06156231, 0.69681795, 0.67383079],
[0.31445525, 0.83595899, 0.705263]]),
np.array([[[0.66830849, 0.35291152, 0.04349641],
[0.62815067, 0.75400669, 0.43327223],
[0.64411265, 0.26457201, 0.84955846],
[0.40414402, 0.6676325, 0.33832645]],
[[0.43584521, 0.66741136, 0.70503737],
[0.26677207, 0.4960918, 0.32260832],
[0.61456562, 0.35535548, 0.83467162],
[0.74859915, 0.70133653, 0.57041632]]]),
np.array([[[0.73339975, 0.78643665, 0.45069885],
[0.74986861, 0.67810782, 0.54938006],
[0.26128632, 0.07936058, 0.02278049],
[0.8836243, 0.62162135, 0.45951421]],
[[0.70346187, 0.10926256, 0.67854529],
[0.39645904, 0.65467032, 0.56780106],
[0.6622787, 0.39268321, 0.60983543],
[0.19075994, 0.36662241, 0.91810201]],
[[0.30012843, 0.60306739, 0.62558339],
[0.3591911, 0.37191895, 0.65078595],
[0.3670538, 0.37250119, 0.55795222],
[0.01902248, 0.72941351, 0.57023361]]]),
np.array([[7, 3, 2, 5, 2, 14, 34, 1, 10, 2, 15, 5]]),
np.array([
0.03280384, 0.94485231, 0.43253641, 0.29359502, 0.84187526,
0.50992254, 0.29902775, 0.48802801, 0.19521612, 0.16601624,
0.95941332, 0.90352063
])
]
writer.write(data_list)
writer.close()
for file_name in file_names:
assert os.path.exists(file_name)
#todo assert data is read correctly
# read_data = gdal.Open(file_name)
# for i, data in enumerate(data_list):
# data = read_data.GetRasterBand(i + 1).ReadArray()
finally:
for file_name in file_names:
if os.path.exists(file_name):
os.remove(file_name)
class InferenceWriter:
def __init__(self,
actual_parameter_list: List[str],
full_parameter_list: List[str],
output_folder: str,
time_step: datetime,
geo_transform: tuple,
projection: str,
width: int,
height: int,
output_format: str = "GeoTiff",
state_folder: str = None):
self._actual_parameter_list = actual_parameter_list
file_names = []
self.param_positions = {}
num_bands = []
for param in actual_parameter_list:
if param in full_parameter_list:
file_names.append(
os.path.join(
output_folder,
"%s_%s.tif" % (param, time_step.strftime("A%Y%j"))))
num_bands.append(1)
file_names.append(
os.path.join(
output_folder, "%s_%s_unc.tif" %
(param, time_step.strftime("A%Y%j"))))
num_bands.append(1)
self.param_positions[param] = full_parameter_list.index(param)
if output_format == "GeoTiff":
self.writer = GeoTiffWriter(file_names,
geo_transform,
projection,
width,
height,
num_bands,
data_types=None)
else:
raise ValueError(f"Format {output_format} is not supported.")
self._width = width
self._height = height
self._offset_x = 0
self._offset_y = 0
self.state_folder = state_folder
def set_tile(self, width: int, height: int, offset_x: int, offset_y: int):
self._width = width
self._height = height
self._offset_x = offset_x
self._offset_y = offset_y
def dump_data(self, time_step: Optional[datetime], x_analysis: np.array,
p_analysis: np.array, p_analysis_inv: sp.coo_matrix,
state_mask: np.array, n_params: int):
data = []
for param in self.param_positions:
index = self.param_positions[param]
param_values = np.zeros(state_mask.shape, dtype=np.float32)
param_values[state_mask] = x_analysis[index::n_params]
data.append(param_values)
param_unc = np.zeros(state_mask.shape, dtype=np.float32)
param_unc[state_mask] = 1. / np.sqrt(
p_analysis_inv.diagonal()[index::n_params])
data.append(param_unc)
self.writer.write(data, self._width, self._height, self._offset_x,
self._offset_y)
def dump_state(self, timestep: datetime, x_analysis: np.array,
p_analysis: np.array, p_analysis_inv: np.array,
state_mask: np.array):
if self.state_folder is not None and os.path.exists(self.state_folder):
# Dump to disk P_analysis_inv as sparse matrix in npz
if p_analysis_inv is not None:
file_name = os.path.join(
self.state_folder,
"P_analysis_inv_%s.npz" % (timestep.strftime("A%Y%j")))
try:
sp.save_npz(file_name, p_analysis_inv)
except:
if os.path.exists(file_name):
os.remove(file_name)
if p_analysis is not None:
file_name = os.path.join(
self.state_folder,
"P_analysis_%s.npz" % (timestep.strftime("A%Y%j")))
np.savez(file_name, p_analysis)
# Dump as well the whole x_analysis in a single vector
file_name = os.path.join(
self.state_folder,
"x_analysis_%s.npz" % (timestep.strftime("A%Y%j")))
np.savez(file_name, x_analysis)
# ... and the state mask ...
file_name = os.path.join(
self.state_folder,
"state_mask_%s.npz" % (timestep.strftime("A%Y%j")))
np.savez(file_name, state_mask)
def close(self):
self.writer.close()