def get_layer_min_max(i_info, layers=[1, 2, 3], rgb=False, block_size=2048): min_max = [] if rgb: layer_min = 999999. layer_max = -999999. for i in range(0, i_info.rows, block_size): n_rows = raster_tools.n_rows_cols(i, block_size, i_info.rows) for j in range(0, i_info.cols, block_size): n_cols = raster_tools.n_rows_cols(j, block_size, i_info.cols) sect = i_info.read(bands2open=layers, i=i, j=j, rows=n_rows, cols=n_cols, d_type='float32') sect = get_luminosity(sect) layer_min = min(layer_min, np.percentile(sect, 1)) layer_max = max(layer_max, np.percentile(sect, 99)) min_max.append((layer_min, layer_max)) else: for lb in layers: layer_min = 999999. layer_max = -999999. for i in range(0, i_info.rows, block_size): n_rows = raster_tools.n_rows_cols(i, block_size, i_info.rows) for j in range(0, i_info.cols, block_size): n_cols = raster_tools.n_rows_cols(j, block_size, i_info.cols) sect = i_info.read(bands2open=lb, i=i, j=j, rows=n_rows, cols=n_cols, d_type='float32') layer_min = min(layer_min, np.percentile(sect, 1)) layer_max = max(layer_max, np.percentile(sect, 99)) min_max.append((layer_min, layer_max)) return min_max
def get_chunk_indices(rows, cols, block_size, chunk_size, scale): index_list = list() for i in range(0, rows, chunk_size - scale - block_size): n_rows = raster_tools.n_rows_cols(i, chunk_size, rows) for j in range(0, cols, chunk_size - scale - block_size): n_cols = raster_tools.n_rows_cols(j, chunk_size, cols) index_list.append((i, i + n_rows, j, j + n_cols)) return index_list
def _section_read_write(section_counter): """ Handles the section reading and writing Args: section_counter (int) """ section_pair = potsi[section_counter - 1] # this_parameter_object_ = this_parameter_object.copy() this_parameter_object_ = copy.copy(param_dict) this_parameter_object_ = sputilities.dict2class(this_parameter_object_) # Get the input image information. with raster_tools.ropen( this_parameter_object_.input_image) as this_image_info: this_parameter_object_.update_info(section_counter=section_counter) # Set the output name. this_parameter_object_ = sputilities.scale_fea_check( this_parameter_object_) # Open the status YAML file. mts_ = sputilities.ManageStatus() # Load the status dictionary mts_.load_status(this_parameter_object_.status_file) # Check file status. if os.path.isfile(this_parameter_object_.out_img): if this_parameter_object_.out_img_base in mts_.status_dict: if this_parameter_object_.trigger in mts_.status_dict[ this_parameter_object_.out_img_base]: # Check every trigger because the # entire file needs to be removed. status_list = [ mts_.status_dict[this_parameter_object_.out_img_base] ['{TR}-{BD}'.format( TR=tr, BD=this_parameter_object_.band_position)] for tr in this_parameter_object_.triggers ] if 'corrupt' in status_list: logger.info('Re-running {} ...'.format( this_parameter_object_.out_img)) # Remove the file on the first trigger # if the file is corrupt. if this_parameter_object_.trigger == this_parameter_object_.triggers[ 0]: os.remove(this_parameter_object_.out_img) mts_.status_dict[this_parameter_object_.out_img_base][ '{TR}-{BD}'.format( TR=this_parameter_object_.trigger, BD=this_parameter_object_.band_position )] = 'incomplete' mts_.dump_status(this_parameter_object_.status_file) elif ('corrupt' not in status_list) and ('incomplete' in status_list): logger.info('Re-running {} ...'.format( this_parameter_object_.out_img)) else: if this_parameter_object_.overwrite: logger.info('Re-running {} ...'.format( this_parameter_object_.out_img)) # Remove the file on the first trigger. if this_parameter_object_.trigger == this_parameter_object_.triggers[ 0]: os.remove(this_parameter_object_.out_img) mts_.status_dict[ this_parameter_object_.out_img_base][ '{TR}-{BD}'.format( TR=this_parameter_object_.trigger, BD=this_parameter_object_.band_position )] = 'incomplete' mts_.dump_status( this_parameter_object_.status_file) else: logger.info('{} is already finished ...'.format( this_parameter_object_.out_img)) return else: # Remove the file on the first trigger. if this_parameter_object_.trigger == this_parameter_object_.triggers[ 0]: os.remove(this_parameter_object_.out_img) logger.info('Re-running {} ...'.format( this_parameter_object_.out_img)) i_sect = section_pair[0] j_sect = section_pair[1] # Row and column section bounds checking n_rows = raster_tools.n_rows_cols(i_sect, this_parameter_object_.sect_row_size, this_image_info.rows) n_cols = raster_tools.n_rows_cols(j_sect, this_parameter_object_.sect_col_size, this_image_info.cols) # Open the image array. if this_parameter_object_.trigger.upper( ) in this_parameter_object_.spectral_indices: wavelengths = utils.VI_WAVELENGTHS[ this_parameter_object_.trigger.upper()] # Check if the sensor supports the spectral index utils.sensor_wavelength_check(this_parameter_object_.sat_sensor, wavelengths) # Get the band positions needed # to process the spectral index. spectral_bands = utils.get_index_bands( this_parameter_object_.trigger, this_parameter_object_.sat_sensor) sect_in = this_image_info.read(bands2open=spectral_bands, i=i_sect, j=j_sect, rows=n_rows, cols=n_cols, d_type='float32') sect_in[sect_in >= this_parameter_object_. image_max] = this_parameter_object_.image_max sect_in /= this_parameter_object_.image_max vie = VegIndicesEquations(sect_in, chunk_size=-1) sect_in = vie.compute(this_parameter_object_.trigger.upper(), out_type=1) this_parameter_object_.update_info(image_min=0, image_max=1) elif this_parameter_object_.trigger == 'saliency': sect_in = saliency(this_image_info, this_parameter_object_, i_sect, j_sect, n_rows, n_cols) this_parameter_object_.update_info(image_min=0, image_max=255) elif this_parameter_object_.trigger == 'seg': sect_in = this_image_info.read(bands2open=[1, 2, 3], i=i_sect, j=j_sect, rows=n_rows, cols=n_cols) sect_in = segment_image(sect_in, this_parameter_object_) elif this_parameter_object_.trigger == 'grad': if this_image_info.bands >= 3: sect_in = sputilities.convert_rgb2gray( this_image_info, i_sect, j_sect, n_rows, n_cols, this_parameter_object_.sat_sensor)[0] else: sect_in = this_image_info.read( bands2open=this_parameter_object_.band_position, i=i_sect, j=j_sect, rows=n_rows, cols=n_cols) sect_in = get_mag_avg(sect_in) this_parameter_object_.update_info(image_min=0, image_max=30) elif this_parameter_object_.use_rgb and this_parameter_object_.trigger \ not in this_parameter_object_.spectral_indices + ['grad', 'saliency', 'seg']: sect_in = sputilities.convert_rgb2gray( this_image_info, i_sect, j_sect, n_rows, n_cols, this_parameter_object_.sat_sensor)[0] else: sect_in = this_image_info.read( bands2open=this_parameter_object_.band_position, i=i_sect, j=j_sect, rows=n_rows, cols=n_cols) if this_parameter_object_.trigger == 'dmp': # The Differential Morphological Profile # is a [D x M x N] array # where, # D = the opening/closing derivative. sect_in = get_dmp(sect_in, this_parameter_object_.image_min, this_parameter_object_.image_max) if this_parameter_object_.trigger == 'gabor': sect_in = convolve_gabor(sect_in, this_parameter_object_.image_min, this_parameter_object_.image_max, this_parameter_object_.scales) if this_parameter_object_.trigger == 'orb': sect_in = get_orb_keypoints(sect_in, this_parameter_object_.image_min, this_parameter_object_.image_max) this_parameter_object_.update_info(i_sect_blk_ctr=1, j_sect_blk_ctr=1) if this_parameter_object_.trigger in ['dmp', 'gabor']: l_rows, l_cols = sect_in[0].shape else: l_rows, l_cols = sect_in.shape # Compute section statistics. section_stats_array = spsplit.get_section_stats( sect_in, l_rows, l_cols, this_parameter_object_, section_counter) # Get the section output rows and columns. out_rows, out_cols = spsplit.get_out_dims(l_rows, l_cols, this_parameter_object_) # Reshape the list of features into # <features x rows x columns> array. out_section_array = spreshape.reshape_feature_list( section_stats_array, out_rows, out_cols, this_parameter_object_) is_corrupt = _write_section2file(this_parameter_object_, this_image_info, out_section_array, i_sect, j_sect, out_rows, out_cols, section_counter) this_parameter_object_ = None this_image_info_ = None return is_corrupt
def _block_func(self): global d_stack, forward, backward, label_ones, n_samples, n_steps, n_labels n_steps = self.n_steps n_labels = self.n_labels if self.method == 'forward-backward': forward = np.empty((self.n_steps, self.n_labels), dtype='float32') backward = np.empty((self.n_steps, self.n_labels), dtype='float32') label_ones = np.ones(self.n_labels, dtype='float32') for i in range(0, self.rows, self.block_size): n_rows = raster_tools.n_rows_cols(i, self.block_size, self.rows) for j in range(0, self.cols, self.block_size): hmm_block_tracker = self.out_blocks.replace( '_BLOCK', '{:04d}_{:04d}'.format(i, j)) if os.path.isfile(hmm_block_tracker): continue n_cols = raster_tools.n_rows_cols(j, self.block_size, self.cols) # Total samples in the block. n_samples = n_rows * n_cols # Setup the block stack. # time steps x class layers x rows x columns d_stack = np.empty( (self.n_steps, self.n_labels, n_rows, n_cols), dtype='float32') block_max = 0 # Load the block stack. # *all time steps + all probability layers @ 1 pixel = d_stack[:, :, 0, 0] for step in range(0, self.n_steps): step_array = self.image_infos[step].read( bands2open=self.n_jobs, i=i, j=j, rows=n_rows, cols=n_cols, d_type='float32') # step_array /= step_array.max(axis=0) step_array[np.isnan(step_array) | np.isinf(step_array)] = 0 block_max = max(block_max, step_array.max()) d_stack[step] = step_array if block_max == 0: continue d_stack = d_stack.ravel() # Process each pixel, getting 1 # pixel for all time steps. # # Reshape data to a NxK matrix, # where N is number of time steps and # K is the number of labels. # # Therefore, each row represents one time step. pool = multi.Pool(processes=self.n_jobs) hmm_results = pool.map(self.methods[self.method], range(0, n_samples)) pool.close() pool = None # Parallel(n_jobs=self.n_jobs, # max_nbytes=None)(delayed(self.methods[self.method])(n_sample, # n_samples, # self.n_steps, # self.n_labels) # for n_sample in range(0, n_samples)) hmm_results = np.asarray(hmm_results, dtype='float32').T.reshape( self.n_steps, self.n_labels, n_rows, n_cols) # Reshape the results. # d_stack = d_stack.reshape(self.n_steps, self.n_labels, n_rows, n_cols) # Write the block results to file. # Iterate over each time step. for step in range(0, self.n_steps): # Get the image for the # current time step. out_rst = self.o_infos[step] # Get the array for the # current time step. hmm_sub = hmm_results[step] if self.assign_class: probabilities_argmax = hmm_sub.argmax(axis=0) if isinstance(self.class_list, list): predictions = np.zeros(probabilities_argmax.shape, dtype='uint8') for class_index, real_class in enumerate( self.class_list): predictions[probabilities_argmax == class_index] = real_class else: predictions = probabilities_argmax out_rst.write_array(predictions, i=i, j=j, band=1) out_rst.close_band() else: # Iterate over each probability layer. for layer in range(0, self.n_labels): # Write the block for the # current probability layer. out_rst.write_array(hmm_sub[layer], i=i, j=j, band=layer + 1) out_rst.close_band() with open(hmm_block_tracker, 'wb') as btxt: btxt.write('complete') self.close() out_rst = None
def raster_calc(output, equation=None, out_type='byte', extent=None, overwrite=False, be_quiet=False, out_no_data=0, row_block_size=2000, col_block_size=2000, apply_all_bands=False, **kwargs): """ Raster calculator Args: output (str): The output image. equation (Optional[str]): The equation to calculate. out_type (Optional[str]): The output raster storage type. Default is 'byte'. extent (Optional[str]): An image or instance of ``mappy.ropen`` to use for the output extent. Default is None. overwrite (Optional[bool]): Whether to overwrite an existing IDW image. Default is False. be_quiet (Optional[bool]): Whether to be quiet and do not report progress. Default is False. out_no_data (Optional[int]): The output no data value. Default is 0. row_block_size (Optional[int]): The row block chunk size. Default is 2000. col_block_size (Optional[int]): The column block chunk size. Default is 2000. apply_all_bands (Optional[bool]): Whether to apply the equation to all bands. Default is False. **kwargs (str): The rasters to compute. E.g., A='/some_raster1.tif', F='/some_raster2.tif'. Band positions default to 1 unless given as [A]_band. Examples: >>> from mpglue.raster_calc import raster_calc >>> >>> # Multiply image A x image B >>> raster_calc('/output.tif', >>> equation='A * B', >>> A='/some_raster1.tif', >>> B='some_raster2.tif') >>> >>> # Reads as... >>> # Where image A equals 1 AND image B is greater than 5, >>> # THEN write image A, OTHERWISE write 0 >>> raster_calc('/output.tif', >>> equation='where((A == 1) & (B > 5), A, 0)', >>> A='/some_raster1.tif', >>> B='some_raster2.tif') >>> >>> # Use different bands from the same image. The letter given for the >>> # image must be the same for the band, followed by _band. >>> # E.g., for raster 'n', the corresponding band would be 'n_band'. For >>> # raster 'r', the corresponding band would be 'r_band', etc. >>> raster_calc('/output.tif', >>> equation='(n - r) / (n + r)', >>> n='/some_raster.tif', >>> n_band=4, >>> r='/some_raster.tif', >>> r_band=3) Returns: None, writes to ``output``. """ # Set the image dictionary image_dict = dict() info_dict = dict() info_list = list() band_dict = dict() temp_files = list() if isinstance(extent, str): ot_info = raster_tools.ropen(extent) temp_dict = copy(kwargs) for kw, vw in viewitems(kwargs): if isinstance(vw, str): d_name, f_name = os.path.split(vw) f_base, __ = os.path.splitext(f_name) vw_sub = os.path.join(d_name, '{}_temp.vrt'.format(f_base)) raster_tools.translate(vw, vw_sub, format='VRT', projWin=[ ot_info.left, ot_info.top, ot_info.right, ot_info.bottom ]) temp_files.append(vw_sub) temp_dict[kw] = vw_sub kwargs = temp_dict for kw, vw in viewitems(kwargs): if '_band' not in kw: band_dict['{}_band'.format(kw)] = 1 if isinstance(vw, str): image_dict[kw] = vw exec('i_info_{} = raster_tools.ropen(r"{}")'.format(kw, vw)) exec('info_dict["{}"] = i_info_{}'.format(kw, kw)) exec('info_list.append(i_info_{})'.format(kw)) if isinstance(vw, int): band_dict[kw] = vw for key, value in viewitems(image_dict): equation = equation.replace(key, 'marrvar_{}'.format(key)) # Check for NumPy functions. # for np_func in dir(np): # # if 'np.' + np_func in equation: # # equation = 'np.{}'.format(equation) # break for kw, vw in viewitems(info_dict): o_info = copy(vw) break n_bands = 1 if not apply_all_bands else o_info.bands if isinstance(extent, raster_tools.ropen): # Set the extent from an object. overlap_info = extent elif isinstance(extent, str): # Set the extent from an existing image. overlap_info = raster_tools.ropen(extent) else: # Check overlapping extent overlap_info = info_list[0].copy() for i_ in range(1, len(info_list)): # Get the minimum overlapping extent # from all input images. overlap_info = raster_tools.GetMinExtent(overlap_info, info_list[i_]) o_info.update_info(left=overlap_info.left, right=overlap_info.right, top=overlap_info.top, bottom=overlap_info.bottom, rows=overlap_info.rows, cols=overlap_info.cols, storage=out_type, bands=n_bands) if overwrite: overwrite_file(output) out_rst = raster_tools.create_raster(output, o_info) if n_bands == 1: out_rst.get_band(1) block_rows, block_cols = raster_tools.block_dimensions( o_info.rows, o_info.cols, row_block_size=row_block_size, col_block_size=col_block_size) if not be_quiet: ctr, pbar = _iteration_parameters(o_info.rows, o_info.cols, block_rows, block_cols) # Iterate over the minimum overlapping extent. for i in range(0, o_info.rows, block_rows): n_rows = raster_tools.n_rows_cols(i, block_rows, o_info.rows) for j in range(0, o_info.cols, block_cols): n_cols = raster_tools.n_rows_cols(j, block_cols, o_info.cols) # For each image, get the offset and # convert bands in the equation to ndarrays. for key, value in viewitems(image_dict): # exec 'x_off, y_off = vector_tools.get_xy_offsets3(overlap_info, i_info_{})'.format(key) x_off, y_off = vector_tools.get_xy_offsets( image_info=info_dict[key], x=overlap_info.left, y=overlap_info.top, check_position=False)[2:] exec( 'marrvar_{KEY} = info_dict["{KEY}"].read(bands2open=band_dict["{KEY}_band"], i=i+y_off, j=j+x_off, rows=n_rows, cols=n_cols, d_type="float32")' .format(KEY=key)) if '&&' in equation: out_array = np.empty((n_bands, n_rows, n_cols), dtype='float32') for eqidx, equation_ in enumerate(equation.split('&&')): if 'nan_to_num' in equation_: if not equation_.startswith('np.'): equation_ = 'np.' + equation_ equation_ = 'out_array[eqidx] = {}'.format(equation_) exec(equation_) else: out_array[eqidx] = ne.evaluate(equation_) else: if 'nan_to_num' in equation: equation_ = 'out_array = {}'.format(equation) exec(equation_) else: out_array = ne.evaluate(equation) # Set the output no data values. out_array[np.isnan(out_array) | np.isinf(out_array)] = out_no_data if n_bands == 1: out_rst.write_array(out_array, i=i, j=j) else: for lidx in range(0, n_bands): out_rst.write_array(out_array[lidx], i=i, j=j, band=lidx + 1) if not be_quiet: pbar.update(ctr) ctr += 1 if not be_quiet: pbar.finish() # Close the input image. for key, value in viewitems(info_dict): info_dict[key].close() # close the output drivers out_rst.close_all() out_rst = None # Cleanup for temp_file in temp_files: if os.path.isfile(temp_file): os.remove(temp_file)