def get_layer_min_max(i_info, layers=[1, 2, 3], rgb=False, block_size=2048):
min_max = []
if rgb:
layer_min = 999999.
layer_max = -999999.
for i in range(0, i_info.rows, block_size):
n_rows = raster_tools.n_rows_cols(i, block_size, i_info.rows)
for j in range(0, i_info.cols, block_size):
n_cols = raster_tools.n_rows_cols(j, block_size, i_info.cols)
sect = i_info.read(bands2open=layers,
i=i,
j=j,
rows=n_rows,
cols=n_cols,
d_type='float32')
sect = get_luminosity(sect)
layer_min = min(layer_min, np.percentile(sect, 1))
layer_max = max(layer_max, np.percentile(sect, 99))
min_max.append((layer_min, layer_max))
else:
for lb in layers:
layer_min = 999999.
layer_max = -999999.
for i in range(0, i_info.rows, block_size):
n_rows = raster_tools.n_rows_cols(i, block_size, i_info.rows)
for j in range(0, i_info.cols, block_size):
n_cols = raster_tools.n_rows_cols(j, block_size,
i_info.cols)
sect = i_info.read(bands2open=lb,
i=i,
j=j,
rows=n_rows,
cols=n_cols,
d_type='float32')
layer_min = min(layer_min, np.percentile(sect, 1))
layer_max = max(layer_max, np.percentile(sect, 99))
min_max.append((layer_min, layer_max))
return min_max
def get_chunk_indices(rows, cols, block_size, chunk_size, scale):
index_list = list()
for i in range(0, rows, chunk_size - scale - block_size):
n_rows = raster_tools.n_rows_cols(i, chunk_size, rows)
for j in range(0, cols, chunk_size - scale - block_size):
n_cols = raster_tools.n_rows_cols(j, chunk_size, cols)
index_list.append((i, i + n_rows, j, j + n_cols))
return index_list
def _section_read_write(section_counter):
"""
Handles the section reading and writing
Args:
section_counter (int)
"""
section_pair = potsi[section_counter - 1]
# this_parameter_object_ = this_parameter_object.copy()
this_parameter_object_ = copy.copy(param_dict)
this_parameter_object_ = sputilities.dict2class(this_parameter_object_)
# Get the input image information.
with raster_tools.ropen(
this_parameter_object_.input_image) as this_image_info:
this_parameter_object_.update_info(section_counter=section_counter)
# Set the output name.
this_parameter_object_ = sputilities.scale_fea_check(
this_parameter_object_)
# Open the status YAML file.
mts_ = sputilities.ManageStatus()
# Load the status dictionary
mts_.load_status(this_parameter_object_.status_file)
# Check file status.
if os.path.isfile(this_parameter_object_.out_img):
if this_parameter_object_.out_img_base in mts_.status_dict:
if this_parameter_object_.trigger in mts_.status_dict[
this_parameter_object_.out_img_base]:
# Check every trigger because the
# entire file needs to be removed.
status_list = [
mts_.status_dict[this_parameter_object_.out_img_base]
['{TR}-{BD}'.format(
TR=tr, BD=this_parameter_object_.band_position)]
for tr in this_parameter_object_.triggers
]
if 'corrupt' in status_list:
logger.info('Re-running {} ...'.format(
this_parameter_object_.out_img))
# Remove the file on the first trigger
# if the file is corrupt.
if this_parameter_object_.trigger == this_parameter_object_.triggers[
0]:
os.remove(this_parameter_object_.out_img)
mts_.status_dict[this_parameter_object_.out_img_base][
'{TR}-{BD}'.format(
TR=this_parameter_object_.trigger,
BD=this_parameter_object_.band_position
)] = 'incomplete'
mts_.dump_status(this_parameter_object_.status_file)
elif ('corrupt' not in status_list) and ('incomplete'
in status_list):
logger.info('Re-running {} ...'.format(
this_parameter_object_.out_img))
else:
if this_parameter_object_.overwrite:
logger.info('Re-running {} ...'.format(
this_parameter_object_.out_img))
# Remove the file on the first trigger.
if this_parameter_object_.trigger == this_parameter_object_.triggers[
0]:
os.remove(this_parameter_object_.out_img)
mts_.status_dict[
this_parameter_object_.out_img_base][
'{TR}-{BD}'.format(
TR=this_parameter_object_.trigger,
BD=this_parameter_object_.band_position
)] = 'incomplete'
mts_.dump_status(
this_parameter_object_.status_file)
else:
logger.info('{} is already finished ...'.format(
this_parameter_object_.out_img))
return
else:
# Remove the file on the first trigger.
if this_parameter_object_.trigger == this_parameter_object_.triggers[
0]:
os.remove(this_parameter_object_.out_img)
logger.info('Re-running {} ...'.format(
this_parameter_object_.out_img))
i_sect = section_pair[0]
j_sect = section_pair[1]
# Row and column section bounds checking
n_rows = raster_tools.n_rows_cols(i_sect,
this_parameter_object_.sect_row_size,
this_image_info.rows)
n_cols = raster_tools.n_rows_cols(j_sect,
this_parameter_object_.sect_col_size,
this_image_info.cols)
# Open the image array.
if this_parameter_object_.trigger.upper(
) in this_parameter_object_.spectral_indices:
wavelengths = utils.VI_WAVELENGTHS[
this_parameter_object_.trigger.upper()]
# Check if the sensor supports the spectral index
utils.sensor_wavelength_check(this_parameter_object_.sat_sensor,
wavelengths)
# Get the band positions needed
# to process the spectral index.
spectral_bands = utils.get_index_bands(
this_parameter_object_.trigger,
this_parameter_object_.sat_sensor)
sect_in = this_image_info.read(bands2open=spectral_bands,
i=i_sect,
j=j_sect,
rows=n_rows,
cols=n_cols,
d_type='float32')
sect_in[sect_in >= this_parameter_object_.
image_max] = this_parameter_object_.image_max
sect_in /= this_parameter_object_.image_max
vie = VegIndicesEquations(sect_in, chunk_size=-1)
sect_in = vie.compute(this_parameter_object_.trigger.upper(),
out_type=1)
this_parameter_object_.update_info(image_min=0, image_max=1)
elif this_parameter_object_.trigger == 'saliency':
sect_in = saliency(this_image_info, this_parameter_object_, i_sect,
j_sect, n_rows, n_cols)
this_parameter_object_.update_info(image_min=0, image_max=255)
elif this_parameter_object_.trigger == 'seg':
sect_in = this_image_info.read(bands2open=[1, 2, 3],
i=i_sect,
j=j_sect,
rows=n_rows,
cols=n_cols)
sect_in = segment_image(sect_in, this_parameter_object_)
elif this_parameter_object_.trigger == 'grad':
if this_image_info.bands >= 3:
sect_in = sputilities.convert_rgb2gray(
this_image_info, i_sect, j_sect, n_rows, n_cols,
this_parameter_object_.sat_sensor)[0]
else:
sect_in = this_image_info.read(
bands2open=this_parameter_object_.band_position,
i=i_sect,
j=j_sect,
rows=n_rows,
cols=n_cols)
sect_in = get_mag_avg(sect_in)
this_parameter_object_.update_info(image_min=0, image_max=30)
elif this_parameter_object_.use_rgb and this_parameter_object_.trigger \
not in this_parameter_object_.spectral_indices + ['grad', 'saliency', 'seg']:
sect_in = sputilities.convert_rgb2gray(
this_image_info, i_sect, j_sect, n_rows, n_cols,
this_parameter_object_.sat_sensor)[0]
else:
sect_in = this_image_info.read(
bands2open=this_parameter_object_.band_position,
i=i_sect,
j=j_sect,
rows=n_rows,
cols=n_cols)
if this_parameter_object_.trigger == 'dmp':
# The Differential Morphological Profile
# is a [D x M x N] array
# where,
# D = the opening/closing derivative.
sect_in = get_dmp(sect_in, this_parameter_object_.image_min,
this_parameter_object_.image_max)
if this_parameter_object_.trigger == 'gabor':
sect_in = convolve_gabor(sect_in, this_parameter_object_.image_min,
this_parameter_object_.image_max,
this_parameter_object_.scales)
if this_parameter_object_.trigger == 'orb':
sect_in = get_orb_keypoints(sect_in,
this_parameter_object_.image_min,
this_parameter_object_.image_max)
this_parameter_object_.update_info(i_sect_blk_ctr=1, j_sect_blk_ctr=1)
if this_parameter_object_.trigger in ['dmp', 'gabor']:
l_rows, l_cols = sect_in[0].shape
else:
l_rows, l_cols = sect_in.shape
# Compute section statistics.
section_stats_array = spsplit.get_section_stats(
sect_in, l_rows, l_cols, this_parameter_object_, section_counter)
# Get the section output rows and columns.
out_rows, out_cols = spsplit.get_out_dims(l_rows, l_cols,
this_parameter_object_)
# Reshape the list of features into
# <features x rows x columns> array.
out_section_array = spreshape.reshape_feature_list(
section_stats_array, out_rows, out_cols, this_parameter_object_)
is_corrupt = _write_section2file(this_parameter_object_,
this_image_info, out_section_array,
i_sect, j_sect, out_rows, out_cols,
section_counter)
this_parameter_object_ = None
this_image_info_ = None
return is_corrupt
def _block_func(self):
global d_stack, forward, backward, label_ones, n_samples, n_steps, n_labels
n_steps = self.n_steps
n_labels = self.n_labels
if self.method == 'forward-backward':
forward = np.empty((self.n_steps, self.n_labels), dtype='float32')
backward = np.empty((self.n_steps, self.n_labels), dtype='float32')
label_ones = np.ones(self.n_labels, dtype='float32')
for i in range(0, self.rows, self.block_size):
n_rows = raster_tools.n_rows_cols(i, self.block_size, self.rows)
for j in range(0, self.cols, self.block_size):
hmm_block_tracker = self.out_blocks.replace(
'_BLOCK', '{:04d}_{:04d}'.format(i, j))
if os.path.isfile(hmm_block_tracker):
continue
n_cols = raster_tools.n_rows_cols(j, self.block_size,
self.cols)
# Total samples in the block.
n_samples = n_rows * n_cols
# Setup the block stack.
# time steps x class layers x rows x columns
d_stack = np.empty(
(self.n_steps, self.n_labels, n_rows, n_cols),
dtype='float32')
block_max = 0
# Load the block stack.
# *all time steps + all probability layers @ 1 pixel = d_stack[:, :, 0, 0]
for step in range(0, self.n_steps):
step_array = self.image_infos[step].read(
bands2open=self.n_jobs,
i=i,
j=j,
rows=n_rows,
cols=n_cols,
d_type='float32')
# step_array /= step_array.max(axis=0)
step_array[np.isnan(step_array) | np.isinf(step_array)] = 0
block_max = max(block_max, step_array.max())
d_stack[step] = step_array
if block_max == 0:
continue
d_stack = d_stack.ravel()
# Process each pixel, getting 1
# pixel for all time steps.
#
# Reshape data to a NxK matrix,
# where N is number of time steps and
# K is the number of labels.
#
# Therefore, each row represents one time step.
pool = multi.Pool(processes=self.n_jobs)
hmm_results = pool.map(self.methods[self.method],
range(0, n_samples))
pool.close()
pool = None
# Parallel(n_jobs=self.n_jobs,
# max_nbytes=None)(delayed(self.methods[self.method])(n_sample,
# n_samples,
# self.n_steps,
# self.n_labels)
# for n_sample in range(0, n_samples))
hmm_results = np.asarray(hmm_results,
dtype='float32').T.reshape(
self.n_steps, self.n_labels,
n_rows, n_cols)
# Reshape the results.
# d_stack = d_stack.reshape(self.n_steps, self.n_labels, n_rows, n_cols)
# Write the block results to file.
# Iterate over each time step.
for step in range(0, self.n_steps):
# Get the image for the
# current time step.
out_rst = self.o_infos[step]
# Get the array for the
# current time step.
hmm_sub = hmm_results[step]
if self.assign_class:
probabilities_argmax = hmm_sub.argmax(axis=0)
if isinstance(self.class_list, list):
predictions = np.zeros(probabilities_argmax.shape,
dtype='uint8')
for class_index, real_class in enumerate(
self.class_list):
predictions[probabilities_argmax ==
class_index] = real_class
else:
predictions = probabilities_argmax
out_rst.write_array(predictions, i=i, j=j, band=1)
out_rst.close_band()
else:
# Iterate over each probability layer.
for layer in range(0, self.n_labels):
# Write the block for the
# current probability layer.
out_rst.write_array(hmm_sub[layer],
i=i,
j=j,
band=layer + 1)
out_rst.close_band()
with open(hmm_block_tracker, 'wb') as btxt:
btxt.write('complete')
self.close()
out_rst = None
def raster_calc(output,
equation=None,
out_type='byte',
extent=None,
overwrite=False,
be_quiet=False,
out_no_data=0,
row_block_size=2000,
col_block_size=2000,
apply_all_bands=False,
**kwargs):
"""
Raster calculator
Args:
output (str): The output image.
equation (Optional[str]): The equation to calculate.
out_type (Optional[str]): The output raster storage type. Default is 'byte'.
extent (Optional[str]): An image or instance of ``mappy.ropen`` to use for the output extent. Default is None.
overwrite (Optional[bool]): Whether to overwrite an existing IDW image. Default is False.
be_quiet (Optional[bool]): Whether to be quiet and do not report progress. Default is False.
out_no_data (Optional[int]): The output no data value. Default is 0.
row_block_size (Optional[int]): The row block chunk size. Default is 2000.
col_block_size (Optional[int]): The column block chunk size. Default is 2000.
apply_all_bands (Optional[bool]): Whether to apply the equation to all bands. Default is False.
**kwargs (str): The rasters to compute. E.g., A='/some_raster1.tif', F='/some_raster2.tif'.
Band positions default to 1 unless given as [A]_band.
Examples:
>>> from mpglue.raster_calc import raster_calc
>>>
>>> # Multiply image A x image B
>>> raster_calc('/output.tif',
>>> equation='A * B',
>>> A='/some_raster1.tif',
>>> B='some_raster2.tif')
>>>
>>> # Reads as...
>>> # Where image A equals 1 AND image B is greater than 5,
>>> # THEN write image A, OTHERWISE write 0
>>> raster_calc('/output.tif',
>>> equation='where((A == 1) & (B > 5), A, 0)',
>>> A='/some_raster1.tif',
>>> B='some_raster2.tif')
>>>
>>> # Use different bands from the same image. The letter given for the
>>> # image must be the same for the band, followed by _band.
>>> # E.g., for raster 'n', the corresponding band would be 'n_band'. For
>>> # raster 'r', the corresponding band would be 'r_band', etc.
>>> raster_calc('/output.tif',
>>> equation='(n - r) / (n + r)',
>>> n='/some_raster.tif',
>>> n_band=4,
>>> r='/some_raster.tif',
>>> r_band=3)
Returns:
None, writes to ``output``.
"""
# Set the image dictionary
image_dict = dict()
info_dict = dict()
info_list = list()
band_dict = dict()
temp_files = list()
if isinstance(extent, str):
ot_info = raster_tools.ropen(extent)
temp_dict = copy(kwargs)
for kw, vw in viewitems(kwargs):
if isinstance(vw, str):
d_name, f_name = os.path.split(vw)
f_base, __ = os.path.splitext(f_name)
vw_sub = os.path.join(d_name, '{}_temp.vrt'.format(f_base))
raster_tools.translate(vw,
vw_sub,
format='VRT',
projWin=[
ot_info.left, ot_info.top,
ot_info.right, ot_info.bottom
])
temp_files.append(vw_sub)
temp_dict[kw] = vw_sub
kwargs = temp_dict
for kw, vw in viewitems(kwargs):
if '_band' not in kw:
band_dict['{}_band'.format(kw)] = 1
if isinstance(vw, str):
image_dict[kw] = vw
exec('i_info_{} = raster_tools.ropen(r"{}")'.format(kw, vw))
exec('info_dict["{}"] = i_info_{}'.format(kw, kw))
exec('info_list.append(i_info_{})'.format(kw))
if isinstance(vw, int):
band_dict[kw] = vw
for key, value in viewitems(image_dict):
equation = equation.replace(key, 'marrvar_{}'.format(key))
# Check for NumPy functions.
# for np_func in dir(np):
#
# if 'np.' + np_func in equation:
#
# equation = 'np.{}'.format(equation)
# break
for kw, vw in viewitems(info_dict):
o_info = copy(vw)
break
n_bands = 1 if not apply_all_bands else o_info.bands
if isinstance(extent, raster_tools.ropen):
# Set the extent from an object.
overlap_info = extent
elif isinstance(extent, str):
# Set the extent from an existing image.
overlap_info = raster_tools.ropen(extent)
else:
# Check overlapping extent
overlap_info = info_list[0].copy()
for i_ in range(1, len(info_list)):
# Get the minimum overlapping extent
# from all input images.
overlap_info = raster_tools.GetMinExtent(overlap_info,
info_list[i_])
o_info.update_info(left=overlap_info.left,
right=overlap_info.right,
top=overlap_info.top,
bottom=overlap_info.bottom,
rows=overlap_info.rows,
cols=overlap_info.cols,
storage=out_type,
bands=n_bands)
if overwrite:
overwrite_file(output)
out_rst = raster_tools.create_raster(output, o_info)
if n_bands == 1:
out_rst.get_band(1)
block_rows, block_cols = raster_tools.block_dimensions(
o_info.rows,
o_info.cols,
row_block_size=row_block_size,
col_block_size=col_block_size)
if not be_quiet:
ctr, pbar = _iteration_parameters(o_info.rows, o_info.cols, block_rows,
block_cols)
# Iterate over the minimum overlapping extent.
for i in range(0, o_info.rows, block_rows):
n_rows = raster_tools.n_rows_cols(i, block_rows, o_info.rows)
for j in range(0, o_info.cols, block_cols):
n_cols = raster_tools.n_rows_cols(j, block_cols, o_info.cols)
# For each image, get the offset and
# convert bands in the equation to ndarrays.
for key, value in viewitems(image_dict):
# exec 'x_off, y_off = vector_tools.get_xy_offsets3(overlap_info, i_info_{})'.format(key)
x_off, y_off = vector_tools.get_xy_offsets(
image_info=info_dict[key],
x=overlap_info.left,
y=overlap_info.top,
check_position=False)[2:]
exec(
'marrvar_{KEY} = info_dict["{KEY}"].read(bands2open=band_dict["{KEY}_band"], i=i+y_off, j=j+x_off, rows=n_rows, cols=n_cols, d_type="float32")'
.format(KEY=key))
if '&&' in equation:
out_array = np.empty((n_bands, n_rows, n_cols),
dtype='float32')
for eqidx, equation_ in enumerate(equation.split('&&')):
if 'nan_to_num' in equation_:
if not equation_.startswith('np.'):
equation_ = 'np.' + equation_
equation_ = 'out_array[eqidx] = {}'.format(equation_)
exec(equation_)
else:
out_array[eqidx] = ne.evaluate(equation_)
else:
if 'nan_to_num' in equation:
equation_ = 'out_array = {}'.format(equation)
exec(equation_)
else:
out_array = ne.evaluate(equation)
# Set the output no data values.
out_array[np.isnan(out_array) | np.isinf(out_array)] = out_no_data
if n_bands == 1:
out_rst.write_array(out_array, i=i, j=j)
else:
for lidx in range(0, n_bands):
out_rst.write_array(out_array[lidx],
i=i,
j=j,
band=lidx + 1)
if not be_quiet:
pbar.update(ctr)
ctr += 1
if not be_quiet:
pbar.finish()
# Close the input image.
for key, value in viewitems(info_dict):
info_dict[key].close()
# close the output drivers
out_rst.close_all()
out_rst = None
# Cleanup
for temp_file in temp_files:
if os.path.isfile(temp_file):
os.remove(temp_file)
