def load_grid(self, path):
"""
A utility function to load a microphysical medium from file
Parameters
----------
path: str
Path to file.
Returns
-------
grid: shdom.Grid object
The 3D grid of the medium.
Notes
-----
CSV format should be as follows:
# comment line (description)
nx ny nz
dz dy dz z_levels[0] z_levels[1] ... z_levels[nz-1]
ix iy iz lwc[ix, iy, iz] reff[ix, iy, iz]
.
.
.
ix iy iz lwc[ix, iy, iz] reff[ix, iy, iz]
"""
nx, ny, nz = np.genfromtxt(path, max_rows=1, dtype=int)
dx, dy = np.genfromtxt(path,
max_rows=1,
usecols=(0, 1),
dtype=float,
skip_header=2)
z_grid = np.genfromtxt(path,
max_rows=1,
usecols=range(2, 2 + nz),
dtype=float,
skip_header=2)
x_grid = np.linspace(0.0, (nx - 1) * dx, nx, dtype=np.float32)
y_grid = np.linspace(0.0, (ny - 1) * dy, ny, dtype=np.float32)
grid = shdom.Grid(x=x_grid, y=y_grid, z=z_grid)
return grid
def compute_extinction(self, veff=0.1):
# extrarc grid data:
bounding_box = shdom.BoundingBox(self.bounding_box_xmin,
self.bounding_box_ymin,
self.bounding_box_xmin,
self.bounding_box_xmax,
self.bounding_box_ymax,
self.bounding_box_zmax)
grid = shdom.Grid(bounding_box=bounding_box,
nx=self.nx,
ny=self.ny,
nz=self.nz)
# self.extinction = np.zeros([self.nx,self.ny,self.nz])
veff = veff * np.ones_like(self.re)
lwc = shdom.GridData(grid, self.lwc).squeeze_dims()
reff = shdom.GridData(grid, self.re).squeeze_dims()
veff = shdom.GridData(grid, veff).squeeze_dims()
extinction = self.mie.get_extinction(lwc, reff, veff)
self.extinction = extinction.data
def get_medium_estimator(self, measurements):
"""
Generate the medium estimator for optimization.
Parameters
----------
measurements: shdom.AirMSPIMeasurements
The acquired measurements.
ground_truth: shdom.Scatterer
The ground truth scatterer
Returns
-------
medium_estimator: shdom.MediumEstimator
A medium estimator object which defines the optimized parameters.
"""
wavelength = measurements.wavelength
# if not isinstance(wavelength, list):
# wavelength = [wavelength]
# Define the grid for reconstruction
extinction_grid = albedo_grid = phase_grid = shdom.Grid(
bounding_box=measurements.bb,
nx=self.args.nx,
ny=self.args.ny,
nz=self.args.nz)
grid = extinction_grid + albedo_grid + phase_grid
# Find a cloud mask for non-cloudy grid points
carver = shdom.SpaceCarver(measurements)
mask = carver.carve(grid,
agreement=0.7,
thresholds=self.args.radiance_threshold)
# show_mask = 1
# if show_mask:
# a = (mask.data).astype(int)
# shdom.cloud_plot(a)
# Define the known albedo and phase: either ground-truth or specified, but it is not optimized.
if self.args.use_forward_albedo is False or self.args.use_forward_phase is False:
table_path = self.args.mie_base_path.replace(
'<wavelength>', '{}'.format(shdom.int_round(wavelength)))
self.cloud_generator.add_mie(table_path)
albedo = self.cloud_generator.get_albedo(wavelength, albedo_grid)
phase = self.cloud_generator.get_phase(wavelength, phase_grid)
extinction = shdom.GridDataEstimator(
self.cloud_generator.get_extinction(grid=grid),
min_bound=1e-3,
max_bound=2e2)
cloud_estimator = shdom.OpticalScattererEstimator(
wavelength, extinction, albedo, phase)
cloud_estimator.set_mask(mask)
# Create a medium estimator object (optional Rayleigh scattering)
medium_estimator = shdom.MediumEstimator()
if self.args.add_rayleigh:
air = self.air_generator.get_scatterer(wavelength)
medium_estimator.set_grid(cloud_estimator.grid + air.grid)
medium_estimator.add_scatterer(air, 'air')
else:
medium_estimator.set_grid(cloud_estimator.grid)
medium_estimator.add_scatterer(cloud_estimator, self.scatterer_name)
return medium_estimator
def get_results(self):
"""
"""
self.parse_arguments()
ground_truth, dynamic_solver, measurements = self.load_forward_model(
self.args.input_dir)
wavelength = measurements.wavelength
# Define the grid for reconstruction
if self.args.use_forward_grid:
dynamic_grid = []
for i in range(ground_truth.num_scatterers):
dynamic_grid.append(ground_truth.get_extinction()[i].grid)
grid = dynamic_grid[0]
grid = shdom.Grid(x=grid.x - grid.xmin,
y=grid.y - grid.ymin,
z=grid.z)
else:
extinction_grid = albedo_grid = phase_grid = self.cloud_generator.get_grid(
)
grid = extinction_grid
if self.args.use_forward_cloud_velocity:
cloud_velocity = ground_truth.get_velocity()
cloud_velocity = cloud_velocity[0] * 1000
cloud_velocity[1] = -6
else:
cloud_velocity = None
# Find a cloud mask for non-cloudy grid points
if self.args.use_forward_mask:
mask_list = ground_truth.get_mask(threshold=0.001)
else:
dynamic_carver = shdom.DynamicSpaceCarver(measurements)
mask_list, dynamic_grid, cloud_velocity = dynamic_carver.carve(
grid,
agreement=0.8,
time_list=measurements.time_list,
thresholds=self.args.radiance_threshold,
vx_max=5,
vy_max=0,
gt_velocity=cloud_velocity)
show_mask = 1
if show_mask:
a = (mask_list[0].data).astype(int)
b = ((ground_truth.get_mask(threshold=0.001)[4].resample(
dynamic_grid[4]).data)).astype(int)
print(np.sum((a > b)))
print(np.sum((a < b)))
shdom.cloud_plot(a)
shdom.cloud_plot(b)
if self.args.use_forward_albedo:
albedo = ground_truth.get_albedo()
else:
# albedo = self.cloud_generator.get_albedo(wavelength, albedo_grid)
NotImplemented()
if self.args.use_forward_phase:
phase = ground_truth.get_phase()
else:
NotImplemented()
# phase = self.cloud_generator.get_phase(wavelength, phase.grid)
extinction = shdom.DynamicGridDataEstimator(
ground_truth.get_extinction(dynamic_grid=dynamic_grid),
min_bound=1e-3,
max_bound=2e2)
# if self.args.use_forward_grid:
# extinction = shdom.DynamicGridDataEstimator(ground_truth.get_extinction(),
# min_bound=1e-3,
# max_bound=2e2)
# else:
# if self.args.use_forward_mask:
# grid = ground_truth.get_extinction()[0].grid
# grid = shdom.Grid(x=grid.x - grid.xmin, y=grid.y - grid.ymin, z=grid.z)
# dynamic_carver = shdom.DynamicSpaceCarver(measurements)
# _, dynamic_grid, _ = dynamic_carver.carve(grid, agreement=0.8,
# time_list=ground_truth.time_list,
# thresholds=self.args.radiance_threshold)
# else:
# dynamic_extinction = []
# for grid, ext in zip(dynamic_grid,ground_truth.get_extinction()):
# dynamic_extinction.append(shdom.GridData(grid.grid,ext.data))
# extinction = shdom.DynamicGridDataEstimator(dynamic_extinction,
# min_bound=1e-3,
# max_bound=2e2)
cloud_estimator = shdom.DynamicScattererEstimator(
wavelength,
extinction,
albedo,
phase,
time_list=measurements.time_list)
cloud_estimator.set_mask(mask_list)
# Create a medium estimator object (optional Rayleigh scattering)
air = self.air_generator.get_scatterer(wavelength)
medium_estimator = shdom.DynamicMediumEstimator(
cloud_estimator, air, cloud_velocity)
return medium_estimator
def get_medium_estimator(self,
measurements,
ground_truth,
num_of_mediums,
coarse_extinction=None):
"""
"""
wavelength = measurements.wavelength
# Define the grid for reconstruction
if self.args.use_forward_grid:
#TODO
dynamic_grid = []
for i in range(0, ground_truth.num_scatterers,
int(ground_truth.num_scatterers / num_of_mediums)):
combined_grid = ground_truth.get_extinction()[i].grid
for j in range(
1, int(ground_truth.num_scatterers / num_of_mediums)):
combined_grid = combined_grid + ground_truth.get_extinction(
)[i + j].grid
dynamic_grid.append(combined_grid)
grid = dynamic_grid[0]
grid = shdom.Grid(x=grid.x - grid.xmin,
y=grid.y - grid.ymin,
z=grid.z)
else:
extinction_grid = albedo_grid = phase_grid = self.cloud_generator.get_grid(
)
grid = extinction_grid + albedo_grid + phase_grid
dynamic_grid = [grid] * num_of_mediums
if self.args.use_forward_cloud_velocity:
if ground_truth.num_scatterers > 1:
cloud_velocity = ground_truth.get_velocity()
cloud_velocity = cloud_velocity[0] * 1000 #km/sec to m/sec
else:
cloud_velocity = [0, 0, 0]
else:
cloud_velocity = None
# Find a cloud mask for non-cloudy grid points
if self.args.use_forward_mask:
mask_list = ground_truth.get_mask(threshold=0.000001)
a = (mask_list[0].data).astype(int)
shdom.cloud_plot(a)
else:
dynamic_carver = shdom.DynamicSpaceCarver(measurements)
mask_list, dynamic_grid, cloud_velocity = dynamic_carver.carve(
grid,
agreement=0.9,
time_list=measurements.time_list,
thresholds=self.args.radiance_threshold,
vx_max=5,
vy_max=0,
gt_velocity=cloud_velocity)
show_mask = 1
if show_mask:
a = (mask_list[0].data).astype(int)
b = ((ground_truth.get_mask(threshold=0.0000001)[0].resample(
dynamic_grid[0]).data)).astype(int)
print(np.sum((a > b)))
print(np.sum((a < b)))
shdom.cloud_plot(a)
shdom.cloud_plot(b)
if self.args.use_forward_albedo:
albedo = ground_truth.get_albedo()
else:
# albedo = self.cloud_generator.get_albedo(wavelength, albedo_grid)
NotImplemented()
if self.args.use_forward_phase:
phase = ground_truth.get_phase()
else:
NotImplemented()
# phase = self.cloud_generator.get_phase(wavelength, phase.grid)
# extinction = shdom.DynamicGridDataEstimator(ground_truth.get_extinction(dynamic_grid=dynamic_grid),
# init_val=self.args.extinction,
# min_bound=1e-5,
# max_bound=2e2)
if coarse_extinction is None:
extinction = shdom.DynamicGridDataEstimator(
self.cloud_generator.get_extinction(measurements.wavelength,
dynamic_grid),
min_bound=1e-5,
max_bound=2e2)
else:
previous_num_of_mediums = len(coarse_extinction)
extinction_list = []
for ext in coarse_extinction:
for i in range(int(num_of_mediums / previous_num_of_mediums)):
extinction_list.append(ext)
extinction = shdom.DynamicGridDataEstimator(extinction_list,
min_bound=1e-5,
max_bound=2e2)
kw_optical_scatterer = {
"extinction": extinction,
"albedo": albedo,
"phase": phase
}
cloud_estimator = shdom.DynamicScattererEstimator(
wavelength=wavelength,
time_list=measurements.time_list,
**kw_optical_scatterer)
cloud_estimator.set_mask(mask_list)
# Create a medium estimator object (optional Rayleigh scattering)
air = self.air_generator.get_scatterer(wavelength)
medium_estimator = shdom.DynamicMediumEstimator(
cloud_estimator, air.resample(grid), cloud_velocity)
return medium_estimator
def get_medium_estimator(self, measurements, ground_truth):
"""
"""
wavelength = measurements.wavelength
# Define the grid for reconstruction
if self.args.use_forward_grid:
dynamic_grid = []
for temporary_scatterer in ground_truth._temporary_scatterer_list:
dynamic_grid.append(temporary_scatterer.scatterer.grid)
grid = dynamic_grid[0]
grid = shdom.Grid(x = grid.x - grid.xmin, y = grid.y - grid.ymin, z = grid.z)
else:
extinction_grid = albedo_grid = phase_grid = self.cloud_generator.get_grid()
grid = extinction_grid + albedo_grid + phase_grid
dynamic_grid = [grid] * ground_truth.num_scatterers
if self.args.use_forward_cloud_velocity:
cloud_velocity = ground_truth.get_velocity()
cloud_velocity = cloud_velocity[0]*1000 #km/sec to m/sec
else:
cloud_velocity = None
# Find a cloud mask for non-cloudy grid points
if self.args.use_forward_mask:
mask_list = ground_truth.get_mask(threshold=0.000001)
a = (mask_list[0].data).astype(int)
shdom.cloud_plot(a)
else:
dynamic_carver = shdom.DynamicSpaceCarver(measurements)
mask_list, dynamic_grid, cloud_velocity = dynamic_carver.carve(grid, agreement=0.9,
time_list = measurements.time_list, thresholds=self.args.radiance_threshold,
vx_max = 5, vy_max=0, gt_velocity = cloud_velocity)
show_mask=1
if show_mask:
a = (mask_list[0].data).astype(int)
b = ((ground_truth.get_mask(threshold=0.0000001)[0].resample(dynamic_grid[0]).data)).astype(int)
print(np.sum((a > b)))
print(np.sum((a < b)))
shdom.cloud_plot(a)
shdom.cloud_plot(b)
if self.args.use_forward_albedo:
albedo = ground_truth.get_albedo()
else:
# albedo = self.cloud_generator.get_albedo(wavelength, albedo_grid)
NotImplemented()
if self.args.use_forward_phase:
phase = ground_truth.get_phase()
else:
NotImplemented()
time_list = measurements.time_list
cv_index = self.args.use_cross_validation
if cv_index >= 0:
del dynamic_grid[cv_index]
del mask_list[cv_index]
del albedo[cv_index]
del phase[cv_index]
time_list = np.delete(time_list, cv_index)
extinction = shdom.DynamicGridDataEstimator(self.cloud_generator.get_extinction(measurements.wavelength, dynamic_grid),
min_bound=1e-5,
max_bound=2e2)
kw_optical_scatterer = {"extinction": extinction, "albedo": albedo, "phase": phase}
cloud_estimator = shdom.DynamicScattererEstimator(wavelength=wavelength, time_list=time_list, **kw_optical_scatterer)
cloud_estimator.set_mask(mask_list)
# Create a medium estimator object (optional Rayleigh scattering)
air = self.air_generator.get_scatterer(wavelength)
medium_estimator = shdom.DynamicMediumEstimator(cloud_estimator, air.resample(grid),cloud_velocity)
return medium_estimator
def get_medium_estimator(self, measurements):
"""
"""
num_of_mediums = self.args.num_mediums
cv_index = self.args.use_cross_validation
time_list = measurements.time_list
if cv_index >= 0:
time_list = np.delete(time_list, cv_index)
assert isinstance(num_of_mediums, int) and num_of_mediums <= len(time_list)
wavelength = measurements.wavelength
if not isinstance(wavelength,list):
wavelength = [wavelength]
# Define the grid for reconstruction
grid = albedo_grid = phase_grid = shdom.Grid(bounding_box=measurements.bb,nx=self.args.nx,ny=self.args.ny,nz=self.args.nz)
if self.args.assume_moving_cloud:
cloud_velocity = None
else:
cloud_velocity = [0,0,0]
# Find a cloud mask for non-cloudy grid points
dynamic_carver = shdom.DynamicSpaceCarver(measurements)
mask_list, dynamic_grid, cloud_velocity = dynamic_carver.carve(grid, agreement=0.70,
time_list = measurements.time_list, thresholds=self.args.radiance_threshold,
vx_max = 10, vy_max=10, gt_velocity = cloud_velocity)
mask = mask_list[0]
show_mask=1
if show_mask:
a = mask.data.astype(int)
shdom.cloud_plot(a)
print(cloud_velocity)
print(sum(sum(sum(a))))
table_path = self.args.mie_base_path.replace('<wavelength>', '{}'.format(shdom.int_round(wavelength[0])))
self.cloud_generator.add_mie(table_path)
albedo = self.cloud_generator.get_albedo(wavelength[0], [albedo_grid] * num_of_mediums)
phase = self.cloud_generator.get_phase(wavelength[0], [phase_grid] * num_of_mediums)
# cv_index = self.args.use_cross_validation
# if cv_index >= 0:
# # del dynamic_grid[cv_index]
# # del mask_list[cv_index]
# # del albedo[cv_index]
# # del phase[cv_index]
# time_list = np.delete(measurements.time_list, cv_index)
time_list = np.mean(np.split(time_list, num_of_mediums), 1)
extinction = shdom.DynamicGridDataEstimator(self.cloud_generator.get_extinction(wavelength, [grid] * num_of_mediums),
min_bound=1e-5,
max_bound=2e2)
kw_optical_scatterer = {"extinction": extinction, "albedo": albedo, "phase": phase}
cloud_estimator = shdom.DynamicScattererEstimator(wavelength=wavelength, time_list=time_list, **kw_optical_scatterer)
cloud_estimator.set_mask([mask] * num_of_mediums)
# Create a medium estimator object (optional Rayleigh scattering)
air = self.air_generator.get_scatterer(wavelength)
medium_estimator = shdom.DynamicMediumEstimator(cloud_estimator, air, cloud_velocity)
return medium_estimator