def __init__(self, config_file, row_column='', level='INFO', logfile=None): # Explicitly set the number of threads to be 1, so we more effectively # run in parallel os.environ["MKL_NUM_THREADS"] = "1" # Set logging level self.loglevel = level self.logfile = logfile logging.basicConfig(format='%(levelname)s:%(message)s', level=self.loglevel, filename=self.logfile) self.rows = None self.cols = None self.config = None # Load configuration file self.config = configs.create_new_config(config_file) self.config.get_config_errors() # Initialize ray for parallel execution rayargs = {'address': self.config.implementation.ip_head, '_redis_password': self.config.implementation.redis_password, 'ignore_reinit_error': self.config.implementation.ray_ignore_reinit_error, '_temp_dir': self.config.implementation.ray_temp_dir, 'local_mode': self.config.implementation.n_cores == 1} # We can only set the num_cpus if running on a single-node if self.config.implementation.ip_head is None and self.config.implementation.redis_password is None: rayargs['num_cpus'] = self.config.implementation.n_cores if self.config.implementation.debug_mode is False: ray.init(**rayargs) self.workers = None
def __init__(self, config_file, row_column='', level='INFO', logfile=None): # Explicitly set the number of threads to be 1, so we more effectively #run in parallel os.environ["MKL_NUM_THREADS"] = "1" # Set logging level self.loglevel = level self.logfile = logfile logging.basicConfig(format='%(levelname)s:%(message)s', level=self.loglevel, filename=self.logfile) self.rows = None self.cols = None self.config = None self.fm = None self.iv = None self.io = None self.states = None # Load configuration file self.config = configs.create_new_config(config_file) self.config.get_config_errors() # Initialize ray for parallel execution rayargs = {'address': self.config.implementation.ip_head, 'redis_password': self.config.implementation.redis_password, 'ignore_reinit_error':True, 'local_mode': self.config.implementation.n_cores == 1} # only specify a temporary directory if we are not connecting to # a ray cluster if rayargs['local_mode']: rayargs['temp_dir'] = self.config.implementation.ray_temp_dir # Used to run on a VPN ray.services.get_node_ip_address = lambda: '127.0.0.1' # We can only set the num_cpus if running on a single-node if self.config.implementation.ip_head is None and self.config.implementation.redis_password is None: rayargs['num_cpus'] = self.config.implementation.n_cores ray.init(**rayargs) if len(row_column) > 0: ranges = row_column.split(',') if len(ranges) == 1: self.rows, self.cols = [int(ranges[0])], None if len(ranges) == 2: row_start, row_end = ranges self.rows, self.cols = range( int(row_start), int(row_end)), None elif len(ranges) == 4: row_start, row_end, col_start, col_end = ranges line_start, line_end, samp_start, samp_end = ranges self.rows = range(int(row_start), int(row_end)) self.cols = range(int(col_start), int(col_end)) # Build the forward model and inversion objects self._init_nonpicklable_objects() self.io = IO(self.config, self.fm, self.iv, self.rows, self.cols)
def run_forward(): """Simulate the remote measurement of a spectrally uniform surface.""" # Configure the surface/atmosphere/instrument model testdir, fname = os.path.split(os.path.abspath(__file__)) datadir = os.path.join(testdir, 'data') config = create_new_config(os.path.join(datadir, 'config_forward.json')) fm = ForwardModel(config) iv = Inversion(config, fm) io = IO(config, fm, iv, [0], [0]) # Simulate a measurement and write result for row, col, meas, geom, configs in io: states = iv.invert(meas, geom) io.write_spectrum(row, col, states, meas, geom) assert True return states[0]
def __init__(self, config_file, row_column='', level='INFO', logfile=None): # Explicitly set the number of threads to be 1, so we more effectively # run in parallel os.environ["MKL_NUM_THREADS"] = "1" # Set logging level self.loglevel = level self.logfile = logfile logging.basicConfig(format='%(levelname)s:%(message)s', level=self.loglevel, filename=self.logfile) self.rows = None self.cols = None self.config = None # Load configuration file self.config = configs.create_new_config(config_file) self.config.get_config_errors() # Initialize ray for parallel execution rayargs = { 'address': self.config.implementation.ip_head, '_redis_password': self.config.implementation.redis_password, 'ignore_reinit_error': self.config.implementation.ray_ignore_reinit_error, 'local_mode': self.config.implementation.n_cores == 1 } # only specify a temporary directory if we are not connecting to # a ray cluster if rayargs['local_mode']: rayargs['_temp_dir'] = self.config.implementation.ray_temp_dir # Used to run on a VPN ray.services.get_node_ip_address = lambda: '127.0.0.1' # We can only set the num_cpus if running on a single-node if self.config.implementation.ip_head is None and self.config.implementation.redis_password is None: rayargs['num_cpus'] = self.config.implementation.n_cores ray.init(**rayargs) self.workers = None
def run_inverse(): """Invert the remote measurement.""" # Configure the surface/atmosphere/instrument model testdir, fname = os.path.split(os.path.abspath(__file__)) datadir = os.path.join(testdir, 'data') config = create_new_config(os.path.join(datadir, 'config_forward.json')) fm = ForwardModel(config) iv = Inversion(config, fm) io = IO(config, fm, iv, [0], [0]) geom = None # Get our measurement from the simulation results, and invert. # Calculate uncertainties at the solution state, write result for row, col, meas, geom, configs in io: states = iv.invert(meas, geom) io.write_spectrum(row, col, states, meas, geom) assert True return states[-1]
def run_forward(): """Simulate the remote measurement of a spectrally uniform surface.""" # Configure the surface/atmosphere/instrument model testdir, fname = os.path.split(os.path.abspath(__file__)) datadir = os.path.join(testdir, 'data') config = create_new_config(os.path.join(datadir, 'config_forward.json')) fm = ForwardModel(config) iv = Inversion(config, fm) io = IO(config, fm) # Simulate a measurement and write result for row in range(io.n_rows): for col in range(io.n_cols): id = io.get_components_at_index(row, col) if id is not None: states = iv.invert(id.meas, id.geom) io.write_spectrum(row, col, states, fm, iv) assert True return states[0]
def run_inverse(): """Invert the remote measurement.""" # Configure the surface/atmosphere/instrument model testdir, fname = os.path.split(os.path.abspath(__file__)) datadir = os.path.join(testdir, 'data') config = create_new_config(os.path.join(datadir, 'config_forward.json')) fm = ForwardModel(config) iv = Inversion(config, fm) io = IO(config, fm) # Get our measurement from the simulation results, and invert. # Calculate uncertainties at the solution state, write result for row in range(io.n_rows): for col in range(io.n_cols): id = io.get_components_at_index(row, col) if id is not None: states = iv.invert(id.meas, id.geom) io.write_spectrum(row, col, states, fm, iv) assert True return states[-1]
def _run_chunk(start_line: int, stop_line: int, reference_radiance_file: str, reference_reflectance_file: str, reference_uncertainty_file: str, reference_locations_file: str, input_radiance_file: str, input_locations_file: str, segmentation_file: str, isofit_config: str, output_reflectance_file: str, output_uncertainty_file: str, radiance_factors: np.array, nneighbors: int, nodata_value: float, loglevel: str, logfile: str) -> None: """ Args: start_line: line to start empirical line run at stop_line: line to stop empirical line run at reference_radiance_file: source file for radiance (interpolation built from this) reference_reflectance_file: source file for reflectance (interpolation built from this) reference_uncertainty_file: source file for uncertainty (interpolation built from this) reference_locations_file: source file for file locations (lon, lat, elev), (interpolation built from this) input_radiance_file: input radiance file (interpolate over this) input_locations_file: input location file (interpolate over this) segmentation_file: input file noting the per-pixel segmentation used isofit_config: path to isofit configuration JSON file output_reflectance_file: location to write output reflectance to output_uncertainty_file: location to write output uncertainty to radiance_factors: radiance adjustment factors nneighbors: number of neighbors to use for interpolation nodata_value: nodata value of input and output loglevel: logging level logfile: logging file Returns: None """ logging.basicConfig(format='%(message)s', level=loglevel, filename=logfile) # Load reference images reference_radiance_img = envi.open(reference_radiance_file + '.hdr', reference_radiance_file) reference_reflectance_img = envi.open(reference_reflectance_file + '.hdr', reference_reflectance_file) reference_uncertainty_img = envi.open(reference_uncertainty_file + '.hdr', reference_uncertainty_file) reference_locations_img = envi.open(reference_locations_file + '.hdr', reference_locations_file) n_reference_lines, n_radiance_bands, n_reference_columns = [ int(reference_radiance_img.metadata[n]) for n in ('lines', 'bands', 'samples') ] n_reference_uncertainty_bands = int( reference_uncertainty_img.metadata['bands']) # Load input images input_radiance_img = envi.open(input_radiance_file + '.hdr', input_radiance_file) n_input_lines, n_input_bands, n_input_samples = [ int(input_radiance_img.metadata[n]) for n in ('lines', 'bands', 'samples') ] input_locations_img = envi.open(input_locations_file + '.hdr', input_locations_file) n_location_bands = int(input_locations_img.metadata['bands']) # Load output images output_reflectance_img = envi.open(output_reflectance_file + '.hdr', output_reflectance_file) output_uncertainty_img = envi.open(output_uncertainty_file + '.hdr', output_uncertainty_file) n_output_reflectance_bands = int(output_reflectance_img.metadata['bands']) n_output_uncertainty_bands = int(output_uncertainty_img.metadata['bands']) # Load reference data reference_locations_mm = reference_locations_img.open_memmap( interleave='bip', writable=False) reference_locations = np.array(reference_locations_mm[:, :, :]).reshape( (n_reference_lines, n_location_bands)) reference_radiance_mm = reference_radiance_img.open_memmap( interleave='bip', writable=False) reference_radiance = np.array(reference_radiance_mm[:, :, :]).reshape( (n_reference_lines, n_radiance_bands)) reference_reflectance_mm = reference_reflectance_img.open_memmap( interleave='bip', writable=False) reference_reflectance = np.array( reference_reflectance_mm[:, :, :]).reshape( (n_reference_lines, n_radiance_bands)) reference_uncertainty_mm = reference_uncertainty_img.open_memmap( interleave='bip', writable=False) reference_uncertainty = np.array( reference_uncertainty_mm[:, :, :]).reshape( (n_reference_lines, n_reference_uncertainty_bands)) reference_uncertainty = reference_uncertainty[:, : n_radiance_bands].reshape( (n_reference_lines, n_radiance_bands)) # Load segmentation data if segmentation_file: segmentation_img = envi.open(segmentation_file + '.hdr', segmentation_file) segmentation_img = segmentation_img.read_band(0) else: segmentation_img = None # Prepare instrument model, if available if isofit_config is not None: config = configs.create_new_config(isofit_config) instrument = Instrument(config) logging.info('Loading instrument') # Make sure the instrument is configured for single-pixel noise (no averaging) instrument.integrations = 1 else: instrument = None # Load radiance factors if radiance_factors is None: radiance_adjustment = np.ones(n_radiance_bands, ) else: radiance_adjustment = np.loadtxt(radiance_factors) # Load Tree loc_scaling = np.array([1e5, 1e5, 0.1]) scaled_ref_loc = reference_locations * loc_scaling tree = KDTree(scaled_ref_loc) # Assume (heuristically) that, for distance purposes, 1 m vertically is # comparable to 10 m horizontally, and that there are 100 km per latitude # degree. This is all approximate of course. Elevation appears in the # Third element, and the first two are latitude/longitude coordinates # Iterate through image hash_table = {} for row in np.arange(start_line, stop_line): # Load inline input data input_radiance_mm = input_radiance_img.open_memmap(interleave='bip', writable=False) input_radiance = np.array(input_radiance_mm[row, :, :]) input_radiance = input_radiance * radiance_adjustment input_locations_mm = input_locations_img.open_memmap(interleave='bip', writable=False) input_locations = np.array(input_locations_mm[row, :, :]) output_reflectance_row = np.zeros(input_radiance.shape) + nodata_value output_uncertainty_row = np.zeros(input_radiance.shape) + nodata_value nspectra, start = 0, time.time() for col in np.arange(n_input_samples): x = input_radiance[col, :] if np.all(np.isclose(x, nodata_value)): output_reflectance_row[col, :] = nodata_value output_uncertainty_row[col, :] = nodata_value continue bhat = None if segmentation_img is not None: hash_idx = segmentation_img[row, col] if hash_idx in hash_table: bhat, bmarg, bcov = hash_table[hash_idx] else: loc = reference_locations[ np.array(hash_idx, dtype=int), :] * loc_scaling else: loc = input_locations[col, :] * loc_scaling if bhat is None: dists, nn = tree.query(loc, nneighbors) xv = reference_radiance[nn, :] yv = reference_reflectance[nn, :] uv = reference_uncertainty[nn, :] bhat = np.zeros((n_radiance_bands, 2)) bmarg = np.zeros((n_radiance_bands, 2)) bcov = np.zeros((n_radiance_bands, 2, 2)) for i in np.arange(n_radiance_bands): use = yv[:, i] > 0 n = sum(use) X = np.concatenate((np.ones((n, 1)), xv[use, i:i + 1]), axis=1) W = np.diag(np.ones(n)) # /uv[use, i]) y = yv[use, i:i + 1] try: bhat[i, :] = (inv(X.T @ W @ X) @ X.T @ W @ y).T bcov[i, :, :] = inv(X.T @ W @ X) except: bhat[i, :] = 0 bcov[i, :, :] = 0 bmarg[i, :] = np.diag(bcov[i, :, :]) if (segmentation_img is not None) and not (hash_idx in hash_table): hash_table[hash_idx] = bhat, bmarg, bcov A = np.array((np.ones(n_radiance_bands), x)) output_reflectance_row[col, :] = (np.multiply(bhat.T, A).sum(axis=0)) # Calculate uncertainties. Sy approximation rather than Seps for # speed, for now... but we do take into account instrument # radiometric uncertainties if instrument is None: output_uncertainty_row[col, :] = np.sqrt( np.multiply(bmarg.T, A).sum(axis=0)) else: Sy = instrument.Sy(x, geom=None) calunc = instrument.bval[:instrument.n_chan] output_uncertainty_row[col, :] = np.sqrt( np.diag(Sy) + pow(calunc * x, 2)) * bhat[:, 1] # if loglevel == 'DEBUG': # plot_example(xv, yv, bhat) nspectra = nspectra + 1 elapsed = float(time.time() - start) logging.info('row {}/{}, ({}/{} local), {} spectra per second'.format( row, n_input_lines, int(row - start_line), int(stop_line - start_line), round(float(nspectra) / elapsed, 2))) del input_locations_mm del input_radiance_mm output_reflectance_row = output_reflectance_row.transpose((1, 0)) output_uncertainty_row = output_uncertainty_row.transpose((1, 0)) shp = output_reflectance_row.shape output_reflectance_row = output_reflectance_row.reshape( (1, shp[0], shp[1])) shp = output_uncertainty_row.shape output_uncertainty_row = output_uncertainty_row.reshape( (1, shp[0], shp[1])) _write_bil_chunk( output_reflectance_row, output_reflectance_file, row, (n_input_lines, n_output_reflectance_bands, n_input_samples)) _write_bil_chunk( output_uncertainty_row, output_uncertainty_file, row, (n_input_lines, n_output_uncertainty_bands, n_input_samples))
def empirical_line(reference_radiance_file: str, reference_reflectance_file: str, reference_uncertainty_file: str, reference_locations_file: str, segmentation_file: str, input_radiance_file: str, input_locations_file: str, output_reflectance_file: str, output_uncertainty_file: str, nneighbors: int = 400, nodata_value: float = -9999.0, level: str = 'INFO', logfile: str = None, radiance_factors: np.array = None, isofit_config: str = None, n_cores: int = -1) -> None: """ Perform an empirical line interpolation for reflectance and uncertainty extrapolation Args: reference_radiance_file: source file for radiance (interpolation built from this) reference_reflectance_file: source file for reflectance (interpolation built from this) reference_uncertainty_file: source file for uncertainty (interpolation built from this) reference_locations_file: source file for file locations (lon, lat, elev), (interpolation built from this) segmentation_file: input file noting the per-pixel segmentation used input_radiance_file: input radiance file (interpolate over this) input_locations_file: input location file (interpolate over this) output_reflectance_file: location to write output reflectance to output_uncertainty_file: location to write output uncertainty to nneighbors: number of neighbors to use for interpolation nodata_value: nodata value of input and output level: logging level logfile: logging file radiance_factors: radiance adjustment factors isofit_config: path to isofit configuration JSON file n_cores: number of cores to run on Returns: None """ loglevel = level logging.basicConfig(format='%(message)s', level=loglevel, filename=logfile) # Open input data to check that band formatting is correct # Load reference set radiance reference_radiance_img = envi.open(reference_radiance_file + '.hdr', reference_radiance_file) n_reference_lines, n_radiance_bands, n_reference_columns = [ int(reference_radiance_img.metadata[n]) for n in ('lines', 'bands', 'samples') ] if n_reference_columns != 1: raise IndexError("Reference data should be a single-column list") # Load reference set reflectance reference_reflectance_img = envi.open(reference_reflectance_file + '.hdr', reference_reflectance_file) nrefr, nbr, srefr = [ int(reference_reflectance_img.metadata[n]) for n in ('lines', 'bands', 'samples') ] if nrefr != n_reference_lines or nbr != n_radiance_bands or srefr != n_reference_columns: raise IndexError("Reference file dimension mismatch (reflectance)") # Load reference set uncertainty, assuming reflectance uncertainty is # recoreded in the first n_radiance_bands channels of data reference_uncertainty_img = envi.open(reference_uncertainty_file + '.hdr', reference_uncertainty_file) nrefu, ns, srefu = [ int(reference_uncertainty_img.metadata[n]) for n in ('lines', 'bands', 'samples') ] if nrefu != n_reference_lines or ns < n_radiance_bands or srefu != n_reference_columns: raise IndexError("Reference file dimension mismatch (uncertainty)") # Load reference set locations reference_locations_img = envi.open(reference_locations_file + '.hdr', reference_locations_file) nrefl, lb, ls = [ int(reference_locations_img.metadata[n]) for n in ('lines', 'bands', 'samples') ] if nrefl != n_reference_lines or lb != 3: raise IndexError("Reference file dimension mismatch (locations)") input_radiance_img = envi.open(input_radiance_file + '.hdr', input_radiance_file) n_input_lines, n_input_bands, n_input_samples = [ int(input_radiance_img.metadata[n]) for n in ('lines', 'bands', 'samples') ] if n_radiance_bands != n_input_bands: msg = 'Number of channels mismatch: input (%i) vs. reference (%i)' raise IndexError(msg % (nbr, n_radiance_bands)) input_locations_img = envi.open(input_locations_file + '.hdr', input_locations_file) nll, nlb, nls = [ int(input_locations_img.metadata[n]) for n in ('lines', 'bands', 'samples') ] if nll != n_input_lines or nlb != 3 or nls != n_input_samples: raise IndexError('Input location dimension mismatch') # Create output files output_metadata = input_radiance_img.metadata output_metadata['interleave'] = 'bil' output_reflectance_img = envi.create_image(output_reflectance_file + '.hdr', ext='', metadata=output_metadata, force=True) output_uncertainty_img = envi.create_image(output_uncertainty_file + '.hdr', ext='', metadata=output_metadata, force=True) # Now cleanup inputs and outputs, we'll write dynamically above del output_reflectance_img, output_uncertainty_img del reference_reflectance_img, reference_uncertainty_img, reference_locations_img, input_radiance_img, input_locations_img # Initialize ray cluster start_time = time.time() if isofit_config is not None: iconfig = configs.create_new_config(isofit_config) else: # If none, create a temporary config to get default ray parameters iconfig = configs.Config({}) if n_cores == -1: n_cores = iconfig.implementation.n_cores rayargs = { 'ignore_reinit_error': iconfig.implementation.ray_ignore_reinit_error, 'local_mode': n_cores == 1, "address": iconfig.implementation.ip_head, "_redis_password": iconfig.implementation.redis_password } # only specify a temporary directory if we are not connecting to # a ray cluster if rayargs['local_mode']: rayargs['_temp_dir'] = iconfig.implementation.ray_temp_dir # Used to run on a VPN ray.services.get_node_ip_address = lambda: '127.0.0.1' # We can only set the num_cpus if running on a single-node if iconfig.implementation.ip_head is None and iconfig.implementation.redis_password is None: rayargs['num_cpus'] = n_cores ray.init(**rayargs) atexit.register(ray.shutdown) n_ray_cores = ray.available_resources()["CPU"] n_cores = min(n_ray_cores, n_input_lines) logging.info( 'Beginning empirical line inversions using {} cores'.format(n_cores)) # Break data into sections line_sections = np.linspace(0, n_input_lines, num=int(n_cores + 1), dtype=int) start_time = time.time() # Run the pool (or run serially) results = [] for l in range(len(line_sections) - 1): args = (line_sections[l], line_sections[l + 1], reference_radiance_file, reference_reflectance_file, reference_uncertainty_file, reference_locations_file, input_radiance_file, input_locations_file, segmentation_file, isofit_config, output_reflectance_file, output_uncertainty_file, radiance_factors, nneighbors, nodata_value, level, logfile) results.append(_run_chunk.remote(*args)) _ = ray.get(results) total_time = time.time() - start_time logging.info( 'Parallel empirical line inversions complete. {} s total, {} spectra/s, {} spectra/s/core' .format(total_time, line_sections[-1] * n_input_samples / total_time, line_sections[-1] * n_input_samples / total_time / n_cores))
def main(): # Parse arguments parser = argparse.ArgumentParser(description="built luts for emulation.") parser.add_argument('-ip_head', type=str) parser.add_argument('-redis_password', type=str) parser.add_argument('-n_cores', type=int, default=1) parser.add_argument('-train', type=int, default=1, choices=[0,1]) parser.add_argument('-cleanup', type=int, default=0, choices=[0,1]) args = parser.parse_args() args.train = args.train == 1 args.cleanup = args.cleanup == 1 dayofyear = 200 if args.train: to_solar_zenith_lut = [0, 12.5, 25, 37.5, 50] to_solar_azimuth_lut = [180] to_sensor_azimuth_lut = [180] to_sensor_zenith_lut = [140, 160, 180] altitude_km_lut = [2, 4, 7, 10, 15, 25] elevation_km_lut = [0, 0.75, 1.5, 2.25, 4.5] h2o_lut_grid = np.round(np.linspace(0.1,5,num=5),3).tolist() + [0.6125] h2o_lut_grid.sort() aerfrac_2_lut_grid = np.round(np.linspace(0.01,1,num=5),3).tolist() + [0.5] aerfrac_2_lut_grid.sort() else: # HOLDOUT SET to_solar_zenith_lut = [6, 18, 30,45] to_solar_azimuth_lut = [60, 300] to_sensor_azimuth_lut = [180] to_sensor_zenith_lut = [145, 155, 165, 175] altitude_km_lut = [3, 5.5, 8.5, 12.5, 17.5] elevation_km_lut = [0.325, 1.025, 1.875, 2.575, 4.2] h2o_lut_grid = np.round(np.linspace(0.5,4.5,num=4),3) aerfrac_2_lut_grid = np.round(np.linspace(0.125,0.9,num=4),3) n_lut_build = np.product([len(to_solar_zenith_lut), len(to_solar_azimuth_lut), len(to_sensor_zenith_lut), len(to_sensor_azimuth_lut), len(altitude_km_lut), len(elevation_km_lut), len(h2o_lut_grid), len(aerfrac_2_lut_grid)]) print('Num LUTs to build: {}'.format(n_lut_build)) print('Expected MODTRAN runtime: {} hrs'.format(n_lut_build*1.5)) print('Expected MODTRAN runtime: {} days'.format(n_lut_build*1.5/24)) print('Expected MODTRAN runtime per (40-core) node: {} days'.format(n_lut_build*1.5/24/40)) # Create wavelength file wl = np.arange(0.350, 2.550, 0.0005) wl_file_contents = np.zeros((len(wl),3)) wl_file_contents[:,0] = np.arange(len(wl),dtype=int) wl_file_contents[:,1] = wl wl_file_contents[:,2] = 0.0005 np.savetxt('support/hifidelity_wavelengths.txt',wl_file_contents,fmt='%.5f') # Initialize ray for parallel execution rayargs = {'address': args.ip_head, 'redis_password': args.redis_password, 'local_mode': args.n_cores == 1} if args.n_cores < 40: rayargs['num_cpus'] = args.n_cores ray.init(**rayargs) print(ray.cluster_resources()) template_dir = 'templates' if os.path.isdir(template_dir) is False: os.mkdir(template_dir) modtran_template_file = os.path.join(template_dir,'modtran_template.json') if args.training: isofit_config_file = os.path.join(template_dir,'isofit_template_v2.json') else: isofit_config_file = os.path.join(template_dir,'isofit_template_holdout.json') write_modtran_template(atmosphere_type='ATM_MIDLAT_SUMMER', fid=os.path.splitext(modtran_template_file)[0], altitude_km=altitude_km_lut[0], dayofyear=dayofyear, latitude=to_solar_azimuth_lut[0], longitude=to_solar_zenith_lut[0], to_sensor_azimuth=to_sensor_azimuth_lut[0], to_sensor_zenith=to_sensor_zenith_lut[0], gmtime=0, elevation_km=elevation_km_lut[0], output_file=modtran_template_file) # Make sure H2O grid is fully valid with open(modtran_template_file, 'r') as f: modtran_config = json.load(f) modtran_config['MODTRAN'][0]['MODTRANINPUT']['GEOMETRY']['IPARM'] = 12 modtran_config['MODTRAN'][0]['MODTRANINPUT']['ATMOSPHERE']['H2OOPT'] = '+' modtran_config['MODTRAN'][0]['MODTRANINPUT']['AEROSOLS']['VIS'] = 100 with open(modtran_template_file, 'w') as fout: fout.write(json.dumps(modtran_config, cls=SerialEncoder, indent=4, sort_keys=True)) paths = Paths(os.path.join('.',os.path.basename(modtran_template_file)), args.training) build_main_config(paths, isofit_config_file, to_solar_azimuth_lut, to_solar_zenith_lut, aerfrac_2_lut_grid, h2o_lut_grid, elevation_km_lut, altitude_km_lut, to_sensor_azimuth_lut, to_sensor_zenith_lut, n_cores=args.n_cores) config = configs.create_new_config(isofit_config_file) isofit_modtran = ModtranRT(config.forward_model.radiative_transfer.radiative_transfer_engines[0], config) isofit_sixs = SixSRT(config.forward_model.radiative_transfer.radiative_transfer_engines[1], config) # cleanup if args.cleanup: for to_rm in ['*r_k', '*t_k', '*tp7', '*wrn', '*psc', '*plt', '*7sc', '*acd']: cmd = 'find {os.path.join(paths.lut_modtran_directory)} -name "{to_rm}"') print(cmd) os.system(cmd)
def main(): # Parse arguments parser = argparse.ArgumentParser(description="built luts for emulation.") parser.add_argument('-config_file', type=str, default='templates/isofit_template.json') parser.add_argument('-keys', type=str, default=['transm', 'rhoatm', 'sphalb'], nargs='+') parser.add_argument('-munge_dir', type=str, default='munged') args = parser.parse_args() np.random.seed(13) for key_ind, key in enumerate(args.keys): munge_file = os.path.join(args.munge_dir, key + '.npz') if os.path.isfile(munge_file) is False: config = configs.create_new_config(args.config_file) # Note - this goes way faster if you comment out the Vector Interpolater build section in each of these isofit_modtran = ModtranRT(config.forward_model.radiative_transfer.radiative_transfer_engines[0], config, build_lut = False) isofit_sixs = SixSRT(config.forward_model.radiative_transfer.radiative_transfer_engines[1], config, build_lut = False) sixs_results = get_obj_res(isofit_sixs, key, resample=False) modtran_results = get_obj_res(isofit_modtran, key) if os.path.isdir(os.path.dirname(munge_file) is False): os.mkdir(os.path.dirname(munge_file)) for fn in isofit_modtran.files: mod_output = isofit_modtran.load_rt(fn) sol_irr = mod_output['sol'] if np.all(np.isfinite(sol_irr)): break np.savez(munge_file, modtran_results=modtran_results, sixs_results=sixs_results, sol_irr=sol_irr) modtran_results = None sixs_results = None for key_ind, key in enumerate(args.keys): munge_file = os.path.join(args.munge_dir, key + '.npz') npzf = np.load(munge_file) dim1 = int(np.product(np.array(npzf['modtran_results'].shape)[:-1])) dim2 = npzf['modtran_results'].shape[-1] if modtran_results is None: modtran_results = np.zeros((dim1,dim2*len(args.keys))) modtran_results[:,dim2*key_ind:dim2*(key_ind+1)] = npzf['modtran_results'] dim1 = int(np.product(np.array(npzf['sixs_results'].shape)[:-1])) dim2 = npzf['sixs_results'].shape[-1] if sixs_results is None: sixs_results = np.zeros((dim1,dim2*len(args.keys))) sixs_results[:,dim2*key_ind:dim2*(key_ind+1)] = npzf['sixs_results'] sol_irr = npzf['sol_irr'] config = configs.create_new_config(args.config_file) isofit_modtran = ModtranRT(config.forward_model.radiative_transfer.radiative_transfer_engines[0], config, build_lut=False) isofit_sixs = SixSRT(config.forward_model.radiative_transfer.radiative_transfer_engines[1], config, build_lut=False) sixs_names = isofit_sixs.lut_names modtran_names = isofit_modtran.lut_names if 'elev' in sixs_names: sixs_names[sixs_names.index('elev')] = 'GNDALT' if 'viewzen' in sixs_names: sixs_names[sixs_names.index('viewzen')] = 'OBSZEN' if 'viewaz' in sixs_names: sixs_names[sixs_names.index('viewaz')] = 'TRUEAZ' if 'alt' in sixs_names: sixs_names[sixs_names.index('alt')] = 'H1ALT' if 'AOT550' in sixs_names: sixs_names[sixs_names.index('AOT550')] = 'AERFRAC_2' reorder_sixs = [sixs_names.index(x) for x in modtran_names] points = isofit_modtran.points.copy() points_sixs = isofit_sixs.points.copy()[:,reorder_sixs] if 'OBSZEN' in modtran_names: print('adjusting') points_sixs[:, modtran_names.index('OBSZEN')] = 180 - points_sixs[:, modtran_names.index('OBSZEN')] ind = np.lexsort(tuple([points[:,x] for x in range(points.shape[-1])])) points = points[ind,:] modtran_results = modtran_results[ind,:] ind_sixs = np.lexsort(tuple([points_sixs[:,x] for x in range(points_sixs.shape[-1])])) points_sixs = points_sixs[ind_sixs,:] sixs_results = sixs_results[ind_sixs,:] good_data = np.all(np.isnan(modtran_results) == False,axis=1) good_data[np.any(np.isnan(sixs_results),axis=1)] = False modtran_results = modtran_results[good_data,:] sixs_results = sixs_results[good_data,:] points = points[good_data,...] points_sixs = points_sixs[good_data,...] print(sixs_results.shape) print(modtran_results.shape) tmp = isofit_sixs.load_rt(isofit_sixs.files[0]) np.savez(os.path.join(args.munge_dir, 'combined_training_data.npz'), sixs_results=sixs_results, modtran_results=modtran_results, points=points, points_sixs=points_sixs, keys=args.keys, point_names=modtran_names, modtran_wavelengths=isofit_modtran.wl, sixs_wavelengths=isofit_sixs.grid, sol_irr=sol_irr)
def _run_chunk(start_line: int, stop_line: int, reference_radiance_file: str, reference_atm_file: str, reference_locations_file: str, input_radiance_file: str, input_locations_file: str, segmentation_file: str, isofit_config: dict, output_reflectance_file: str, output_uncertainty_file: str, radiance_factors: np.array, nneighbors: int, nodata_value: float) -> None: """ Args: start_line: line to start empirical line run at stop_line: line to stop empirical line run at reference_radiance_file: source file for radiance (interpolation built from this) reference_atm_file: source file for atmosphere coefficients (interpolation built from this) reference_locations_file: source file for file locations (lon, lat, elev), (interpolation built from this) input_radiance_file: input radiance file (interpolate over this) input_locations_file: input location file (interpolate over this) segmentation_file: input file noting the per-pixel segmentation used isofit_config: dictionary-stype isofit configuration output_reflectance_file: location to write output reflectance to output_uncertainty_file: location to write output uncertainty to radiance_factors: radiance adjustment factors nneighbors: number of neighbors to use for interpolation nodata_value: nodata value of input and output Returns: None """ # Load reference images reference_radiance_img = envi.open(envi_header(reference_radiance_file), reference_radiance_file) reference_atm_img = envi.open(envi_header(reference_atm_file), reference_atm_file) reference_locations_img = envi.open(envi_header(reference_locations_file), reference_locations_file) n_reference_lines, n_radiance_bands, n_reference_columns = [ int(reference_radiance_img.metadata[n]) for n in ('lines', 'bands', 'samples') ] # Load input images input_radiance_img = envi.open(envi_header(input_radiance_file), input_radiance_file) n_input_lines, n_input_bands, n_input_samples = [ int(input_radiance_img.metadata[n]) for n in ('lines', 'bands', 'samples') ] wl = np.array( [float(w) for w in input_radiance_img.metadata['wavelength']]) input_locations_img = envi.open(envi_header(input_locations_file), input_locations_file) n_location_bands = int(input_locations_img.metadata['bands']) # Load output images output_reflectance_img = envi.open(envi_header(output_reflectance_file), output_reflectance_file) output_uncertainty_img = envi.open(envi_header(output_uncertainty_file), output_uncertainty_file) n_output_reflectance_bands = int(output_reflectance_img.metadata['bands']) n_output_uncertainty_bands = int(output_uncertainty_img.metadata['bands']) # Load reference data reference_locations_mm = reference_locations_img.open_memmap( interleave='source', writable=False) reference_locations = np.array(reference_locations_mm[:, :, :]).reshape( (n_reference_lines, n_location_bands)) reference_radiance_mm = reference_radiance_img.open_memmap( interleave='source', writable=False) reference_radiance = np.array(reference_radiance_mm[:, :, :]).reshape( (n_reference_lines, n_radiance_bands)) reference_atm_mm = reference_atm_img.open_memmap(interleave='source', writable=False) reference_atm = np.array(reference_atm_mm[:, :, :]).reshape( (n_reference_lines, n_radiance_bands * 5)) rhoatm = reference_atm[:, :n_radiance_bands] sphalb = reference_atm[:, n_radiance_bands:(n_radiance_bands * 2)] transm = reference_atm[:, (n_radiance_bands * 2):(n_radiance_bands * 3)] solirr = reference_atm[:, (n_radiance_bands * 3):(n_radiance_bands * 4)] coszen = reference_atm[:, (n_radiance_bands * 4):(n_radiance_bands * 5)] # Load segmentation data if segmentation_file: segmentation_img = envi.open(envi_header(segmentation_file), segmentation_file) segmentation_img = segmentation_img.read_band(0) else: segmentation_img = None # Prepare instrument model, if available if isofit_config is not None: config = configs.create_new_config(isofit_config) instrument = Instrument(config) logging.info('Loading instrument') else: instrument = None # Load radiance factors if radiance_factors is None: radiance_adjustment = np.ones(n_radiance_bands, ) else: radiance_adjustment = np.loadtxt(radiance_factors) # PCA coefficients rdn_pca = PCA(n_components=2) reference_pca = rdn_pca.fit_transform(reference_radiance * radiance_adjustment) # Create the tree to find nearest neighbor segments. # Assume (heuristically) that, for distance purposes, 1 m vertically is # comparable to 10 m horizontally, and that there are 100 km per latitude # degree. This is all approximate of course. Elevation appears in the # Third element, and the first two are latitude/longitude coordinates # The fourth and fifth elements are "spectral distance" determined by the # top principal component coefficients loc_scaling = np.array([1e5, 1e5, 10, 100, 100]) scaled_ref_loc = np.concatenate( (reference_locations, reference_pca), axis=1) * loc_scaling tree = KDTree(scaled_ref_loc) # Fit GP parameters on transmissivity of an H2O feature, in the # first 400 datapoints use = np.arange(min(len(rhoatm), 400)) h2oband = np.argmin(abs(wl - 940)) scale = (500, 500, 500, 500, 500) bounds = ((100, 2000), (100, 2000), (100, 2000), (100, 2000), (100, 2000)) kernel = RBF(length_scale=scale, length_scale_bounds=bounds) +\ WhiteKernel(noise_level=0.01, noise_level_bounds=(1e-10, 0.1)) gp = GaussianProcessRegressor(kernel=kernel, alpha=0.0, normalize_y=True) gp = gp.fit(scaled_ref_loc[use, :], transm[use, h2oband]) kernel = gp.kernel_ # Iterate through image. Each segment has its own GP, stored in a # hash table indexed by location in the segmentation map hash_table = {} for row in np.arange(start_line, stop_line): # Load inline input data input_radiance_mm = input_radiance_img.open_memmap(interleave='source', writable=False) input_radiance = np.array(input_radiance_mm[row, :, :]) if input_radiance_img.metadata['interleave'] == 'bil': input_radiance = input_radiance.transpose((1, 0)) input_radiance = input_radiance * radiance_adjustment input_locations_mm = input_locations_img.open_memmap( interleave='source', writable=False) input_locations = np.array(input_locations_mm[row, :, :]) if input_locations_img.metadata['interleave'] == 'bil': input_locations = input_locations.transpose((1, 0)) output_reflectance_row = np.zeros(input_radiance.shape) + nodata_value output_uncertainty_row = np.zeros(input_radiance.shape) + nodata_value nspectra, start = 0, time.time() for col in np.arange(n_input_samples): # Get radiance, pca coordinates, physical location for this datum my_rdn = input_radiance[col, :] my_pca = rdn_pca.transform(my_rdn[np.newaxis, :]) my_loc = np.r_[input_locations[col, :], my_pca[0, :]] * loc_scaling if np.all(np.isclose(my_rdn, nodata_value)): output_reflectance_row[col, :] = nodata_value output_uncertainty_row[col, :] = nodata_value continue # Retrieve or build the GP gp_rhoatm, gp_sphalb, gp_transm, irr = None, None, None, None hash_idx = segmentation_img[row, col] if hash_idx in hash_table: gp_rhoatm, gp_sphalb, gp_transm, irr = hash_table[hash_idx] else: # There is no GP for this segment, so we build one from # the atmospheric coefficients from closest neighbors dists, nn = tree.query(my_loc, nneighbors) neighbor_rhoatm = rhoatm[nn, :] neighbor_transm = transm[nn, :] neighbor_sphalb = sphalb[nn, :] neighbor_coszen = coszen[nn, :] neighbor_solirr = solirr[nn, :] neighbor_locs = scaled_ref_loc[nn, :] # Create a new GP using the optimized parameters as a fixed kernel gp_rhoatm = GaussianProcessRegressor(kernel=kernel, alpha=0.0, normalize_y=True, optimizer=None) gp_rhoatm.fit(neighbor_locs, neighbor_rhoatm) gp_sphalb = GaussianProcessRegressor(kernel=kernel, alpha=0.0, normalize_y=True, optimizer=None) gp_sphalb.fit(neighbor_locs, neighbor_sphalb) gp_transm = GaussianProcessRegressor(kernel=kernel, alpha=0.0, normalize_y=True, optimizer=None) gp_transm.fit(neighbor_locs, neighbor_transm) irr = solirr[1, :] * coszen[1, :] irr[irr < 1e-8] = 1e-8 hash_table[hash_idx] = (gp_rhoatm, gp_sphalb, gp_transm, irr) my_rhoatm = gp_rhoatm.predict(my_loc[np.newaxis, :]) my_sphalb = gp_sphalb.predict(my_loc[np.newaxis, :]) my_transm = gp_transm.predict(my_loc[np.newaxis, :]) my_rho = (my_rdn * np.pi) / irr my_rfl = 1.0 / (my_transm / (my_rho - my_rhoatm) + my_sphalb) output_reflectance_row[col, :] = my_rfl # Calculate uncertainties. Sy approximation rather than Seps for # speed, for now... but we do take into account instrument # radiometric uncertainties #output_uncertainty_row[col, :] = np.zeros() #if instrument is None: #else: # Sy = instrument.Sy(x, geom=None) # calunc = instrument.bval[:instrument.n_chan] # output_uncertainty_row[col, :] = np.sqrt( # np.diag(Sy) + pow(calunc * x, 2)) * bhat[:, 1] # if loglevel == 'DEBUG': # plot_example(xv, yv, bhat) nspectra = nspectra + 1 elapsed = float(time.time() - start) logging.info('row {}/{}, ({}/{} local), {} spectra per second'.format( row, n_input_lines, int(row - start_line), int(stop_line - start_line), round(float(nspectra) / elapsed, 2))) del input_locations_mm del input_radiance_mm output_reflectance_row = output_reflectance_row.transpose((1, 0)) output_uncertainty_row = output_uncertainty_row.transpose((1, 0)) shp = output_reflectance_row.shape output_reflectance_row = output_reflectance_row.reshape( (1, shp[0], shp[1])) shp = output_uncertainty_row.shape output_uncertainty_row = output_uncertainty_row.reshape( (1, shp[0], shp[1])) _write_bil_chunk( output_reflectance_row, output_reflectance_file, row, (n_input_lines, n_output_reflectance_bands, n_input_samples)) _write_bil_chunk( output_uncertainty_row, output_uncertainty_file, row, (n_input_lines, n_output_uncertainty_bands, n_input_samples))