def main(infile): inhdr = infile + ".hdr" outhdr = os.path.basename(infile)[0:-11] + "_glint900metric.hdr" img = envi.open(inhdr, infile) wl = s.array([float(w) for w in img.metadata['wavelength']]) if (wl[0] < 100): wl = wl * 1000 fwhm = s.array([float(w) for w in img.metadata['fwhm']]) b900 = s.argmin(abs(wl - 900.0)) ## b1050 = s.argmin(abs(wl-1050.0)) b900imgs = img.read_bands([b900 - 1, b900, b900 + 1]) ## b1050imgs = img.read_bands([b1050-1, b1050, b1050+1]) glintm900 = np.median(b900imgs, axis=2).reshape( (b900imgs.shape[0], 1, b900imgs.shape[1])) ## glintm1050 = np.median(b1050imgs, axis=2).reshape((b1050imgs.shape[0], 1, b1050imgs.shape[1])) # make output glint metric file and open memmap metadata = img.metadata.copy() metadata['bands'] = '%i' % 1 metadata['interleave'] = 'bil' metadata['data type'] = '4' out = envi.create_image(outhdr, metadata, ext='', force=True) outmm = out.open_memmap(interleave='source', writable=True) outmm[:, :, :] = glintm900 del outmm, out, glintm900, img
def main(): # Read in CSV file and convert to BIP ENVI file csv_path = sys.argv[1] output_path = csv_path.replace(".csv", "") output_hdr_path = output_path + ".hdr" spectra_df = pd.read_csv(csv_path, dtype="float32") spectra_df = spectra_df.fillna(-9999) lines, bands = spectra_df.shape wavelengths = spectra_df.keys().values hdr = { "lines": str(lines), "samples": "1", "bands": str(bands), "header offset": "0", "file type": "ENVI Standard", "data type": "4", "interleave": "bip", "byte order": "0", "data ignore value": "-9999", "wavelength": wavelengths } out_file = envi.create_image(output_hdr_path, hdr, ext='', force=True) output_mm = out_file.open_memmap(interleave='source', writable=True) # Iterate through dataframe and write to output memmap for index, row in spectra_df.iterrows(): output_mm[index, 0, :] = row.values # Write to disk del output_mm
def remap(inputfile, labels, outputfile, flag, chunksize): """.""" ref_file = inputfile lbl_file = labels out_file = outputfile nchunk = chunksize ref_img = envi.open(ref_file + '.hdr', ref_file) ref_meta = ref_img.metadata ref_mm = ref_img.open_memmap(interleave='source', writable=False) ref = np.array(ref_mm[:, :]) lbl_img = envi.open(lbl_file + '.hdr', lbl_file) lbl_meta = lbl_img.metadata labels = lbl_img.read_band(0) nl = int(lbl_meta['lines']) ns = int(lbl_meta['samples']) nb = int(ref_meta['bands']) out_meta = dict([(k, v) for k, v in ref_meta.items()]) out_meta["samples"] = ns out_meta["bands"] = nb out_meta["lines"] = nl out_meta['data type'] = ref_meta['data type'] out_meta["interleave"] = "bil" out_img = envi.create_image(out_file + '.hdr', metadata=out_meta, ext='', force=True) out_mm = out_img.open_memmap(interleave='source', writable=True) # Iterate through image "chunks," restoring as we go for lstart in np.arange(0, nl, nchunk): print(lstart) del out_mm out_mm = out_img.open_memmap(interleave='source', writable=True) # Which labels will we extract? ignore zero index lend = min(lstart + nchunk, nl) lbl = labels[lstart:lend, :] out = flag * np.ones((lbl.shape[0], nb, lbl.shape[1])) for row in range(lbl.shape[0]): for col in range(lbl.shape[1]): out[row, :, col] = np.squeeze(ref[int(lbl[row, col]), :]) out_mm[lstart:lend, :, :] = out
def extractions(inputfile, labels, output, chunksize, flag, n_cores: int = 1, ray_address: str = None, ray_redis_password: str = None, ray_temp_dir: str = None, ray_ip_head=None, logfile: str = None, loglevel: str = 'INFO'): """...""" in_file = inputfile lbl_file = labels out_file = output nchunk = chunksize dtm = {'4': np.float32, '5': np.float64} # Open input data, get dimensions in_img = envi.open(in_file + '.hdr', in_file) meta = in_img.metadata nl, nb, ns = [int(meta[n]) for n in ('lines', 'bands', 'samples')] img_mm = in_img.open_memmap(interleave='bip', writable=False) lbl_img = envi.open(lbl_file + '.hdr', lbl_file) labels = lbl_img.read_band(0) un_labels = np.unique(labels).tolist() if 0 not in un_labels: un_labels.insert(0, 0) nout = len(un_labels) # Start up a ray instance for parallel work rayargs = { 'ignore_reinit_error': True, 'local_mode': n_cores == 1, "address": ray_address, "_redis_password": ray_redis_password } if rayargs['local_mode']: rayargs['_temp_dir'] = ray_temp_dir # Used to run on a VPN ray.services.get_node_ip_address = lambda: '127.0.0.1' # We can only set the num_cpus if running on a single-node if ray_ip_head is None and ray_redis_password is None: rayargs['num_cpus'] = n_cores ray.init(**rayargs) atexit.register(ray.shutdown) labelid = ray.put(labels) jobs = [] for lstart in np.arange(0, nl, nchunk): lend = min(lstart + nchunk, nl) jobs.append( extract_chunk.remote(lstart, lend, in_file, labelid, flag, logfile=logfile, loglevel=loglevel)) # Collect results rreturn = [ray.get(jid) for jid in jobs] # Iterate through image "chunks," segmenting as we go out = np.zeros((nout, nb)) for idx, ret in rreturn: if ret is not None: for _output_index in range(len(idx)): out[un_labels.index(idx[_output_index]), ...] = ret[_output_index, ...] del rreturn ray.shutdown() out[np.logical_not(np.isfinite(out))] = flag meta["lines"] = str(nout) meta["bands"] = str(nb) meta["samples"] = '1' meta["interleave"] = "bil" out_img = envi.create_image(out_file + '.hdr', metadata=meta, ext='', force=True) out_mm = np.memmap(out_file, dtype=dtm[meta['data type']], mode='w+', shape=(nout, 1, nb)) if dtm[meta['data type']] == np.float32: out_mm[:, 0, :] = np.array(out, np.float32) else: out_mm[:, 0, :] = np.array(out, np.float64)
#Working startrow = 0 endrow = lat startcol = 0 endcol = long import itertools p = Pool(cpuNode) p.starmap( GGGl, itertools.product(range(startrow, endrow, maxchuckSize), range(startcol, endcol, maxchuckSize), [maxchuckSize], [{ "YAM": True }])) p.close() # In[72]: #Writing output meta = MSAVI.metadata meta["band names"] = outputparams meta['bands'] = str(len(meta["band names"])) meta['data type'] = '4' #del meta['description'] img = envi.create_image(outputName, meta, force=True) mm = img.open_memmap(writable=True) mm[:, :, :] = fp[:, :, :]
def interpolate_atmosphere(reference_radiance_file: str, reference_atm_file: str, reference_locations_file: str, segmentation_file: str, input_radiance_file: str, input_locations_file: str, output_reflectance_file: str, output_uncertainty_file: str, nneighbors: int = 15, nodata_value: float = -9999.0, level: str = 'INFO', radiance_factors: np.array = None, isofit_config: dict = None, n_cores: int = -1) -> None: """ Perform a Gaussian process interpolation of atmospheric parameters. It relies on precalculated atmospheric coefficients at a subset of spatial locations stored in a file. The file has each coefficient defined for every radiance channel, appearing in the order: (1) atmospheric path reflectance; (2) spherical sky albedo; (3) total diffuse and direct transmittance of the two-part downwelling and upwelling path; (4) extraterrestrial solar irradiance; (5) cosine of solar zenith angle. Args: reference_radiance_file: source file for radiance (interpolation built from this) reference_atm_file: source file for atmospheric coefficients (interpolation from this) reference_locations_file: source file for file locations (lon, lat, elev), (interpolation from this) segmentation_file: input file noting the per-pixel segmentation used input_radiance_file: input radiance file (interpolate over this) input_locations_file: input location file (interpolate over this) output_reflectance_file: location to write output reflectance output_uncertainty_file: location to write output uncertainty nneighbors: number of neighbors to use for interpolation nodata_value: nodata value of input and output level: logging level radiance_factors: radiance adjustment factors isofit_config: dictionary-stype isofit configuration n_cores: number of cores to run on Returns: None """ loglevel = level logging.basicConfig(format='%(message)s', level=loglevel) # Open input data to check that band formatting is correct # Load reference set radiance reference_radiance_img = envi.open(envi_header(reference_radiance_file), reference_radiance_file) n_reference_lines, n_radiance_bands, n_reference_columns = [ int(reference_radiance_img.metadata[n]) for n in ('lines', 'bands', 'samples') ] if n_reference_columns != 1: raise IndexError("Reference data should be a single-column list") # Load reference set atmospheric coefficients reference_atm_img = envi.open(envi_header(reference_atm_file), reference_atm_file) nrefa, nba, srefa = [ int(reference_atm_img.metadata[n]) for n in ('lines', 'bands', 'samples') ] if nrefa != n_reference_lines or srefa != n_reference_columns: raise IndexError("Reference file dimension mismatch (atmosphere)") if nba != (n_radiance_bands * 5): raise IndexError( "Reference atmosphere file has incorrect dimensioning") # Load reference set locations reference_locations_img = envi.open(envi_header(reference_locations_file), reference_locations_file) nrefl, lb, ls = [ int(reference_locations_img.metadata[n]) for n in ('lines', 'bands', 'samples') ] if nrefl != n_reference_lines or lb != 3: raise IndexError("Reference file dimension mismatch (locations)") input_radiance_img = envi.open(envi_header(input_radiance_file), input_radiance_file) n_input_lines, n_input_bands, n_input_samples = [ int(input_radiance_img.metadata[n]) for n in ('lines', 'bands', 'samples') ] if n_radiance_bands != n_input_bands: msg = 'Number of channels mismatch: input (%i) vs. reference (%i)' raise IndexError(msg % (n_input_bands, n_radiance_bands)) input_locations_img = envi.open(envi_header(input_locations_file), input_locations_file) nll, nlb, nls = [ int(input_locations_img.metadata[n]) for n in ('lines', 'bands', 'samples') ] if nll != n_input_lines or nlb != 3 or nls != n_input_samples: raise IndexError('Input location dimension mismatch') # Create output files output_metadata = input_radiance_img.metadata output_metadata['interleave'] = 'bil' output_reflectance_img = envi.create_image( envi_header(output_reflectance_file), ext='', metadata=output_metadata, force=True) output_uncertainty_img = envi.create_image( envi_header(output_uncertainty_file), ext='', metadata=output_metadata, force=True) # Now cleanup inputs and outputs, we'll write dynamically above del output_reflectance_img, output_uncertainty_img del reference_atm_img, reference_locations_img, input_radiance_img, input_locations_img # Determine the number of cores to use if n_cores == -1: n_cores = multiprocessing.cpu_count() n_cores = min(n_cores, n_input_lines) # Break data into sections line_sections = np.linspace(0, n_input_lines, num=n_cores + 1, dtype=int) # Set up our pool pool = multiprocessing.Pool(processes=n_cores) start_time = time.time() logging.info( 'Beginning atmospheric interpolation inversions using {} cores'.format( n_cores)) # Run the pool (or run serially) results = [] for l in range(len(line_sections) - 1): args = ( line_sections[l], line_sections[l + 1], reference_radiance_file, reference_atm_file, reference_locations_file, input_radiance_file, input_locations_file, segmentation_file, isofit_config, output_reflectance_file, output_uncertainty_file, radiance_factors, nneighbors, nodata_value, ) if n_cores != 1: results.append(pool.apply_async(_run_chunk, args)) else: _run_chunk(*args) pool.close() pool.join() total_time = time.time() - start_time logging.info( 'Parallel empirical line inversions complete. {} s total, {} spectra/s, {} spectra/s/core' .format(total_time, line_sections[-1] * n_input_samples / total_time, line_sections[-1] * n_input_samples / total_time / n_cores))
def main(): description = 'Spectroscopic Surface & Atmosphere Fitting' parser = argparse.ArgumentParser() parser.add_argument('config_file') parser.add_argument('--row_column', default='') parser.add_argument('--profile', default='') args = parser.parse_args() # Setup config = json.load(open(args.config_file, 'r')) configdir, f = split(abspath(args.config_file)) config = expand_all_paths(config, configdir) fm = ForwardModel(config['forward_model']) iv = Inversion(config['inversion'], fm) out = Output(config, iv) simulation_mode = (not ('input' in config)) or \ (not ('measured_radiance_file' in config['input'])) text_mode = simulation_mode or \ config['input']['measured_radiance_file'].endswith('txt') # Do we apply a radiance correction? if (not simulation_mode) and \ 'radiometry_correction_file' in config['input']: radiance_correction_file = config['input'][ 'radiometry_correction_file'] radiance_correction, wl = spectrumLoad(radiance_correction_file) else: radiance_correction = None if text_mode: # ------------------------------- Text mode # build geometry object if 'input' in config: obs, loc, glt = None, None, None if 'glt_file' in config['input']: glt = s.loadtxt(config['input']['glt_file']) if 'obs_file' in config['input']: obs = s.loadtxt(config['input']['obs_file']) if 'loc_file' in config['input']: loc = s.loadtxt(config['input']['loc_file']) geom = Geometry(obs=obs, glt=glt, loc=loc) else: geom = None if simulation_mode: # Get our measurement from instrument data or the initial state vector state_est = fm.init_val.copy() rdn_est = fm.calc_rdn(state_est, geom) rdn_meas = rdn_est.copy() rdn_sim = fm.instrument.simulate_measurement(rdn_meas, geom) rfl_est, rdn_est, path_est, S_hat, K, G =\ iv.forward_uncertainty(state_est, rdn_meas, geom) else: # Invert instrument measurement rdn_meas, wl = spectrumLoad( config['input']['measured_radiance_file']) if radiance_correction is not None: rdn_meas = rdn_meas * radiance_correction rdn_sim = None if len(args.profile) > 0: cProfile.runctx('iv.invert(rdn_meas, geom, None)', globals(), locals()) sys.exit(0) else: state_est = iv.invert(rdn_meas, geom, out=out) rfl_est, rdn_est, path_est, S_hat, K, G =\ iv.forward_uncertainty(state_est, rdn_meas, geom) out.write_spectrum(state_est, rfl_est, rdn_est, path_est, rdn_meas, rdn_sim, geom) else: # ------------------------------ Binary mode meas_file = config['input']['measured_radiance_file'] meas_hdr = meas_file + '.hdr' meas = envi.open(meas_hdr, meas_file) nl, nb, ns = [ int(meas.metadata[n]) for n in ('lines', 'bands', 'samples') ] # Do we apply a flatfield correction? if 'flatfield_correction_file' in config['input']: ffile = config['input']['flatfield_correction_file'] fcor = envi.open(ffile + '.hdr', ffile) fcor_mm = fcor.open_memmap(interleave='source', writable=False) flatfield = s.array(fcor_mm[0, :, :]) else: flatfield = None if 'obs_file' in config['input']: obs_file = config['input']['obs_file'] obs_hdr = obs_file + '.hdr' obs = envi.open(obs_hdr, obs_file) if int(obs.metadata['bands']) != 11 and \ int(obs.metadata['bands']) != 10: raise ValueError('Expected 10 or 11 bands in OBS file') if int(obs.metadata['lines']) != nl or \ int(obs.metadata['samples']) != ns: raise ValueError('obs file dimensions do not match radiance') else: obs_file, obs_hdr = None, None if 'glt_file' in config['input']: glt_file = config['input']['glt_file'] glt_hdr = glt_file + '.hdr' glt = envi.open(glt_hdr, glt_file) if int(glt.metadata['bands']) != 2: raise ValueError('Expected two bands in GLT file') if int(glt.metadata['lines']) != nl or \ int(glt.metadata['samples']) != ns: raise ValueError('GLT file dimensions do not match radiance') else: glt_file, glt_hdr = None, None if 'loc_file' in config['input']: loc_file = config['input']['loc_file'] loc_hdr = loc_file + '.hdr' loc = envi.open(loc_hdr, loc_file) if int(loc.metadata['bands']) != 3: raise ValueError('Expected three bands in LOC file') if int(loc.metadata['lines']) != nl or \ int(loc.metadata['samples']) != ns: raise ValueError('loc file dimensions do not match radiance') else: loc_file, loc_hdr = None, None rfl_file = config['output']['estimated_reflectance_file'] rfl_hdr = rfl_file + '.hdr' rfl_meta = meas.metadata.copy() if not os.path.exists(rfl_hdr): rfl = envi.create_image(rfl_hdr, rfl_meta, ext='', force=True) del rfl rfl = envi.open(rfl_hdr, rfl_file) state_file = config['output']['estimated_state_file'] state_hdr = state_file + '.hdr' state_meta = meas.metadata.copy() state_meta['bands'] = len(fm.statevec) state_meta['band names'] = fm.statevec[:] if not os.path.exists(state_hdr): state = envi.create_image(state_hdr, state_meta, ext='', force=True) del state state = envi.open(state_hdr, state_file) mdl_file = config['output']['modeled_radiance_file'] mdl_hdr = mdl_file + '.hdr' mdl_meta = meas.metadata.copy() if not os.path.exists(mdl_hdr): mdl = envi.create_image(mdl_hdr, mdl_meta, ext='', force=True) del mdl mdl = envi.open(mdl_hdr, mdl_file) path_file = config['output']['path_radiance_file'] path_hdr = path_file + '.hdr' path_meta = meas.metadata.copy() if not os.path.exists(path_hdr): path = envi.create_image(path_hdr, path_meta, ext='', force=True) del path path = envi.open(path_hdr, path_file) post_file = config['output']['posterior_uncertainty_file'] post_hdr = post_file + '.hdr' post_meta = state_meta.copy() if not os.path.exists(post_hdr): post = envi.create_image(post_hdr, post_meta, ext='', force=True) del post post = envi.open(post_hdr, post_file) if 'component_file' in config['output']: comp_file = config['output']['component_file'] comp_hdr = comp_file + '.hdr' comp_meta = state_meta.copy() comp_meta['bands'] = 1 comp_meta['band names'] = '{Surface Model Component}' if not os.path.exists(comp_hdr): comp = envi.create_image(comp_hdr, comp_meta, ext='', force=True) del comp comp = envi.open(comp_hdr, comp_file) else: comp = None meas_mm, obs_mm, state_mm, rfl_mm, path_mm, mdl_mm = \ None, None, None, None, None, None # Did the user specify a row,column tuple, or a row? if len(args.row_column) < 1: lines, samps = range(nl), range(ns) else: # Restrict the range of the retrieval if overridden. ranges = args.row_column.split(',') if len(ranges) == 1: lines, samps = [int(ranges[0])], range(ns) if len(ranges) == 2: line_start, line_end = ranges lines, samps = range(int(line_start), int(line_end)), range(ns) elif len(ranges) == 4: line_start, line_end, samp_start, samp_end = ranges lines = range(int(line_start), int(line_end)) samps = range(int(samp_start), int(samp_end)) # analyze the image one frame at a time for i in lines: # Flush cache every once in a while print('line %i/%i' % (i, nl)) if meas_mm is None or i % 100 == 0: # Refresh Radiance buffer del meas meas = envi.open(meas_hdr, meas_file) meas_mm = meas.open_memmap(interleave='source', writable=False) # Refresh OBS buffer if obs_hdr is not None: del obs obs = envi.open(obs_hdr, obs_file) obs_mm = obs.open_memmap(interleave='source', writable=False) # Refresh GLT buffer if glt_hdr is not None: del glt glt = envi.open(glt_hdr, glt_file) glt_mm = glt.open_memmap(interleave='source', writable=False) # Refresh LOC buffer if loc_hdr is not None: del loc loc = envi.open(loc_hdr, loc_file) loc_mm = loc.open_memmap(interleave='source', writable=False) # Refresh Output buffers del rfl rfl = envi.open(rfl_hdr, rfl_file) rfl_mm = rfl.open_memmap(interleave='source', writable=True) del state state = envi.open(state_hdr, state_file) state_mm = state.open_memmap(interleave='source', writable=True) del path path = envi.open(path_hdr, path_file) path_mm = path.open_memmap(interleave='source', writable=True) del mdl mdl = envi.open(mdl_hdr, mdl_file) mdl_mm = mdl.open_memmap(interleave='source', writable=True) del post post = envi.open(post_hdr, post_file) post_mm = post.open_memmap(interleave='source', writable=True) if comp is not None: del comp comp = envi.open(comp_hdr, comp_file) comp_mm = comp.open_memmap(interleave='source', writable=True) # translate to BIP meas_frame = s.array(meas_mm[i, :, :]).T if obs_hdr is not None: obs_frame = s.array(obs_mm[i, :, :]) if glt_hdr is not None: glt_frame = s.array(glt_mm[i, :, :]) if loc_hdr is not None: loc_frame = s.array(loc_mm[i, :, :]) init = None nl = int(rfl_meta['lines']) ns = int(rfl_meta['samples']) nb = int(rfl_meta['bands']) nsv = int(state_meta['bands']) if comp is not None: comp_frame = s.zeros((1, ns)) post_frame = s.zeros((nsv, ns), dtype=s.float32) rfl_frame = s.zeros((nb, ns), dtype=s.float32) state_frame = s.zeros((nsv, ns), dtype=s.float32) path_frame = s.zeros((nb, ns), dtype=s.float32) mdl_frame = s.zeros((nb, ns), dtype=s.float32) for j in samps: try: # Use AVIRIS-C convention? if loc_hdr is not None and "t01p00r" in loc_file: loc_frame[:, 2] = loc_frame[:, 2] / \ 1000.0 # translate to km rdn_meas = meas_frame[j, :] # Bad data test if all(rdn_meas < -49.0): raise OOBError() if obs_hdr is not None: obs_spectrum = obs_frame[j, :] else: obs_spectrum = None if glt_hdr is not None: pc = abs(glt_frame[j, 0]) - 1 if pc < 0: raise OOBError() glt_spectrum = glt_frame[j, :] else: pc = j glt_spectrum = None if loc_hdr is not None: loc_spectrum = loc_frame[j, :] else: loc_spectrum = None if flatfield is not None: rdn_meas = rdn_meas * flatfield[:, pc] if radiance_correction is not None: rdn_meas = rdn_meas * radiance_correction geom = Geometry(obs_spectrum, glt_spectrum, loc_spectrum, pushbroom_column=pc) # Inversion if len(args.profile) > 0: cProfile.runctx('iv.invert(rdn_meas, geom, None)', globals(), locals()) sys.exit(0) else: state_est = iv.invert(rdn_meas, geom, None, init=init) rfl_est, rdn_est, path_est, S_hat, K, G =\ iv.forward_uncertainty(state_est, rdn_meas, geom) #init = state_est.copy() # write spectrum state_surf = state_est[iv.fm.surface_inds] post_frame[:, j] = s.sqrt(s.diag(S_hat)) rfl_frame[:, j] = rfl_est state_frame[:, j] = state_est path_frame[:, j] = path_est mdl_frame[:, j] = rdn_est if comp is not None: comp_frame[:, j] = iv.fm.surface.component( state_surf, geom) except OOBError: post_frame[:, j] = -9999 * s.ones((nsv)) rfl_frame[:, j] = -9999 * s.ones((nb)) state_frame[:, j] = -9999 * s.ones((nsv)) path_frame[:, j] = -9999 * s.ones((nb)) mdl_frame[:, j] = -9999 * s.ones((nb)) if comp is not None: comp_frame[:, j] = -9999 post_mm[i, :, :] = post_frame.copy() rfl_mm[i, :, :] = rfl_frame.copy() state_mm[i, :, :] = state_frame.copy() path_mm[i, :, :] = path_frame.copy() mdl_mm[i, :, :] = mdl_frame.copy() if comp is not None: comp_mm[i, :, :] = comp_frame.copy() del post_mm del rfl_mm del state_mm del path_mm del mdl_mm if comp is not None: del comp_mm
def main(): # parse arguments desc = "Break up images into smaller chunks" parser = argparse.ArgumentParser(description=desc) parser.add_argument('input', type=str, metavar='INPUT', help='input reflectance file (ENVI header present)') args = parser.parse_args() s.set_printoptions(precision=4) s.seterr(all='ignore') scenario = args.scenario # Set up input and output file paths in_file = args.input in_hdr = find_header(in_file) suffix = ['a', 'b', 'c', 'd'] img = envi.open(in_hdr, in_file) nl = int(float(img.metadata['lines'])/2.0) ns = int(float(img.metadata['samples'])/2.0) rowbounds = [[0, nl], [0, nl], \ [nl, img.metadata['lines']], [nl, img.metadata['lines']]] colbounds = [[0, ns], [ns, img.metadata['samples']], \ [0, ns], [ns, img.metadata['samples']]] for j in [0,1,2,3]: out_file = os.path.splitext(in_file)[0] + suffix[j] out_file = os.path.splitext(in_file)[0] + suffix[j] + '.hdr' img = envi.open(in_hdr, in_file) envi.read_subregion((0,1), (0,img.shape[1]), use_memmap=False) inmm = img.open_memmap(interleave='source', writable=False) ## wl = s.array([float(w) for w in img.metadata['wavelength']]) ## if (wl[0] < 100): ## wl = wl * 1000 ## fwhm = s.array([float(w) for w in img.metadata['fwhm']]) # set up metadata and constants const = constants(wl, args.mode, scenario, args.root_dir+'/data/') # make output Rb file and open memmap metadata = img.metadata.copy() metadata['bands'] = '%i' % sum(const.use_out) metadata['interleave'] = 'bil' metadata['wavelength'] = ['%7.2f'%w for w in wl[const.use_out]] metadata['data type'] = '2' out = envi.create_image(out_hdr, metadata, ext='',force=True) outmm = out.open_memmap(interleave='source', writable=True) # iterate over rows if args.rows is None: start, fin = 0, nl else: start, fin = [int(f) for f in args.rows.split(',')] for i in range(start, fin): # Flush cache print 'line %i/%i' % (i,nl) if args.depth_file is not None: del bathy bathy= spectral.io.envi.open(args.depth_file+'.hdr', args.depth_file) bathymm = bathy.open_memmap(interleave='source', writable=False) bath = s.array(bathymm[i,:,:]) if bathy.metadata['interleave'] == 'bil': bath = bath.T else: bath = None del img img = spectral.io.envi.open(in_hdr, in_file) inmm = img.open_memmap(interleave='source', writable=False) Rrs = s.array(inmm[i,:,:]) if img.metadata['interleave'] == 'bil': Rrs = Rrs.T if int(img.metadata['data type']) != 4: Rrs = Rrs / s.float32(10000) del out_Rb out_Rb = spectral.io.envi.open(out_Rb_hdr, out_Rb_file) outmm_Rb = out_Rb.open_memmap(interleave='source', writable=True) Rb_frame = s.zeros(outmm_Rb.shape[1:]) del out_Kd out_Kd = spectral.io.envi.open(out_Kd_hdr, out_Kd_file) outmm_Kd = out_Kd.open_memmap(interleave='source', writable=True) Kd_frame = s.zeros(outmm_Kd.shape[1:]) del out_bb out_bb = spectral.io.envi.open(out_bb_hdr, out_bb_file) outmm_bb = out_bb.open_memmap(interleave='source', writable=True) bb_frame = s.zeros(outmm_bb.shape[1:]) del out_dep out_dep = spectral.io.envi.open(out_dep_hdr, out_dep_file) outmm_dep = out_dep.open_memmap(interleave='source', writable=True) dep_frame = s.zeros((outmm_Rb.shape[1],1)) del out_mix out_mix = spectral.io.envi.open(out_mix_hdr, out_mix_file) outmm_mix = out_mix.open_memmap(interleave='source', writable=True) mix_frame = s.zeros(outmm_mix.shape[1:]) # iterate over columns if args.columns is None: colstart, colfin = 0, img.ncols else: colstart, colfin = [int(f) for f in args.columns.split(',')] for col in range(colstart, colfin): if col%100==0: print 'column '+str(col) # check for land and bad data flags if Rrs[col,s.argmin(abs(wl-1000))] > 0.05 or all(Rrs[col,:] <= 0): continue # convert to Rrs Rrs_raw = Rrs[col,:] / s.pi # subtract glint b900 = s.argmin(abs(wl-900.0)) glint = max(0.0001, s.median(Rrs_raw[(b900-2):(b900+3)])) Rrs_raw = Rrs_raw - glint if all(Rrs_raw < 0): continue # out of bounds data # optimize arg = (wl, Rrs_raw, const) state = init_state(wl, Rrs_raw, const) if args.mode != 'aop': print("Only AOP retrievals have pure C support") state, r = leastsq(err, state, args=arg, xtol=1e-6, ftol=1e-6) elif bath is None: state, r = c_leastsq_aop(err,state,args=arg,libaop=libaop) else: b = bath[col,-1] state, r = c_leastsq_aop(err,state,args=arg,libaop=libaop,bath=b) Rrs_model, Kd, bb, R_b, R_std = forward_rrs(wl, Rrs_raw, state, const, est_sd=True) # plot output if args.plot: print('plotting output') err(state, wl, Rrs_raw, const, errbar=None, plot=True) # Transform some parameters out of log space, and Chl-a to mg/m^3 # via the Lee relation water_state = state[:const.nwater] depth_state = state[const.nwater] mix_state = state[(const.nwater+1):] state_phys = s.r_[water_state, depth_state, mix_state] if const.mode=='iop': state_phys[0] = s.exp(s.log(state_phys[0]/0.06)/0.65) # text dump if args.textout is not None: print('writing to '+args.textout) with open(args.textout,'w') as f: f.write('#Depth: %6.2f m\n'%depth_state) f.write('#wl,Rrs.,model,Rb,Rb_sigma_factor,Kd,bb\n') for i in s.where(const.use_out)[0]: f.write('%8.6f,%8.6f,%8.6f,%8.6f,%8.6f,%8.6f,%8.6f\n'%\ (wl[i],Rrs_raw[i],Rrs_model[i],R_b[i],R_std[i],Kd[i],bb[i])) # write output files dep_frame[col,0] = state_phys[const.nbb+const.nKd] bb_frame[col,:] = bb[const.use_out] Kd_frame[col,:] = Kd[const.use_out] Rb_frame[col,:] = R_b[const.use_out] mix_frame[col,:] = mix_state outmm_Rb[i,:,:] = Rb_frame outmm_Kd[i,:,:] = Kd_frame outmm_bb[i,:,:] = bb_frame outmm_mix[i,:,:] = mix_frame outmm_dep[i,:,:] = dep_frame.T if __name__ == "__main__": main()
def segment(spectra: tuple, nodata_value: float, npca: int, segsize: int, nchunk: int, n_cores: int = 1, ray_address: str = None, ray_redis_password: str = None, ray_temp_dir=None, ray_ip_head=None, logfile=None, loglevel='INFO'): """ Segment an image using SLIC on a PCA. Args: spectra: tuple of filepaths of image to segment and (optionally) output label file nodata_value: data to ignore in radiance image npca: number of pca components to use segsize: mean segmentation size nchunk: size of each image chunk n_cores: number of cores to use ray_address: ray address to connect to (for multinode implementation) ray_redis_password: ray password to use (for multinode implementation) ray_temp_dir: ray temp directory to reference ray_ip_head: ray ip head to reference (for multinode use) logfile: logging file to output to loglevel: logging level to use """ logging.basicConfig(format='%(levelname)s:%(message)s', level=loglevel, filename=logfile) in_file = spectra[0] if len(spectra) > 1 and type(spectra) is tuple: lbl_file = spectra[1] else: lbl_file = spectra + '_lbl' # Open input data, get dimensions in_img = envi.open(in_file + '.hdr', in_file) meta = in_img.metadata nl, nb, ns = [int(meta[n]) for n in ('lines', 'bands', 'samples')] # Start up a ray instance for parallel work rayargs = { 'ignore_reinit_error': True, 'local_mode': n_cores == 1, "address": ray_address, "redis_password": ray_redis_password } if rayargs['local_mode']: rayargs['temp_dir'] = ray_temp_dir # Used to run on a VPN ray.services.get_node_ip_address = lambda: '127.0.0.1' # We can only set the num_cpus if running on a single-node if ray_ip_head is None and ray_redis_password is None: rayargs['num_cpus'] = n_cores ray.init(**rayargs) atexit.register(ray.shutdown) # Iterate through image "chunks," segmenting as we go next_label = 1 all_labels = np.zeros((nl, ns), dtype=np.int64) jobs = [] for lstart in np.arange(0, nl, nchunk): # Extract data lend = min(lstart + nchunk, nl) jobs.append( segment_chunk.remote(lstart, lend, in_file, nodata_value, npca, segsize, logfile=logfile, loglevel=loglevel)) # Collect results, making sure each chunk is distinct (note, label indexing is arbitrary, # so not attempt at sorting is made) rreturn = [ray.get(jid) for jid in jobs] for lstart, lend, ret in rreturn: if ret is not None: logging.debug(f'Collecting chunk: {lstart}') chunk_label = ret.copy() chunk_label[chunk_label != 0] += np.max(all_labels) all_labels[lstart:lend, ...] = chunk_label del rreturn ray.shutdown() # Final file I/O logging.debug('Writing output') lbl_meta = { "samples": str(ns), "lines": str(nl), "bands": "1", "header offset": "0", "file type": "ENVI Standard", "data type": "4", "interleave": "bil" } lbl_img = envi.create_image(lbl_file + '.hdr', lbl_meta, ext='', force=True) lbl_mm = lbl_img.open_memmap(interleave='source', writable=True) lbl_mm[:, :] = np.array(all_labels, dtype=np.float32).reshape((nl, 1, ns)) del lbl_mm
def empirical_line(reference_radiance_file: str, reference_reflectance_file: str, reference_uncertainty_file: str, reference_locations_file: str, segmentation_file: str, input_radiance_file: str, input_locations_file: str, output_reflectance_file: str, output_uncertainty_file: str, nneighbors: int = 400, nodata_value: float = -9999.0, level: str = 'INFO', logfile: str = None, radiance_factors: np.array = None, isofit_config: str = None, n_cores: int = -1) -> None: """ Perform an empirical line interpolation for reflectance and uncertainty extrapolation Args: reference_radiance_file: source file for radiance (interpolation built from this) reference_reflectance_file: source file for reflectance (interpolation built from this) reference_uncertainty_file: source file for uncertainty (interpolation built from this) reference_locations_file: source file for file locations (lon, lat, elev), (interpolation built from this) segmentation_file: input file noting the per-pixel segmentation used input_radiance_file: input radiance file (interpolate over this) input_locations_file: input location file (interpolate over this) output_reflectance_file: location to write output reflectance to output_uncertainty_file: location to write output uncertainty to nneighbors: number of neighbors to use for interpolation nodata_value: nodata value of input and output level: logging level logfile: logging file radiance_factors: radiance adjustment factors isofit_config: path to isofit configuration JSON file n_cores: number of cores to run on Returns: None """ loglevel = level logging.basicConfig(format='%(message)s', level=loglevel, filename=logfile) # Open input data to check that band formatting is correct # Load reference set radiance reference_radiance_img = envi.open(reference_radiance_file + '.hdr', reference_radiance_file) n_reference_lines, n_radiance_bands, n_reference_columns = [ int(reference_radiance_img.metadata[n]) for n in ('lines', 'bands', 'samples') ] if n_reference_columns != 1: raise IndexError("Reference data should be a single-column list") # Load reference set reflectance reference_reflectance_img = envi.open(reference_reflectance_file + '.hdr', reference_reflectance_file) nrefr, nbr, srefr = [ int(reference_reflectance_img.metadata[n]) for n in ('lines', 'bands', 'samples') ] if nrefr != n_reference_lines or nbr != n_radiance_bands or srefr != n_reference_columns: raise IndexError("Reference file dimension mismatch (reflectance)") # Load reference set uncertainty, assuming reflectance uncertainty is # recoreded in the first n_radiance_bands channels of data reference_uncertainty_img = envi.open(reference_uncertainty_file + '.hdr', reference_uncertainty_file) nrefu, ns, srefu = [ int(reference_uncertainty_img.metadata[n]) for n in ('lines', 'bands', 'samples') ] if nrefu != n_reference_lines or ns < n_radiance_bands or srefu != n_reference_columns: raise IndexError("Reference file dimension mismatch (uncertainty)") # Load reference set locations reference_locations_img = envi.open(reference_locations_file + '.hdr', reference_locations_file) nrefl, lb, ls = [ int(reference_locations_img.metadata[n]) for n in ('lines', 'bands', 'samples') ] if nrefl != n_reference_lines or lb != 3: raise IndexError("Reference file dimension mismatch (locations)") input_radiance_img = envi.open(input_radiance_file + '.hdr', input_radiance_file) n_input_lines, n_input_bands, n_input_samples = [ int(input_radiance_img.metadata[n]) for n in ('lines', 'bands', 'samples') ] if n_radiance_bands != n_input_bands: msg = 'Number of channels mismatch: input (%i) vs. reference (%i)' raise IndexError(msg % (nbr, n_radiance_bands)) input_locations_img = envi.open(input_locations_file + '.hdr', input_locations_file) nll, nlb, nls = [ int(input_locations_img.metadata[n]) for n in ('lines', 'bands', 'samples') ] if nll != n_input_lines or nlb != 3 or nls != n_input_samples: raise IndexError('Input location dimension mismatch') # Create output files output_metadata = input_radiance_img.metadata output_metadata['interleave'] = 'bil' output_reflectance_img = envi.create_image(output_reflectance_file + '.hdr', ext='', metadata=output_metadata, force=True) output_uncertainty_img = envi.create_image(output_uncertainty_file + '.hdr', ext='', metadata=output_metadata, force=True) # Now cleanup inputs and outputs, we'll write dynamically above del output_reflectance_img, output_uncertainty_img del reference_reflectance_img, reference_uncertainty_img, reference_locations_img, input_radiance_img, input_locations_img # Initialize ray cluster start_time = time.time() if isofit_config is not None: iconfig = configs.create_new_config(isofit_config) else: # If none, create a temporary config to get default ray parameters iconfig = configs.Config({}) if n_cores == -1: n_cores = iconfig.implementation.n_cores rayargs = { 'ignore_reinit_error': iconfig.implementation.ray_ignore_reinit_error, 'local_mode': n_cores == 1, "address": iconfig.implementation.ip_head, "_redis_password": iconfig.implementation.redis_password } # only specify a temporary directory if we are not connecting to # a ray cluster if rayargs['local_mode']: rayargs['_temp_dir'] = iconfig.implementation.ray_temp_dir # Used to run on a VPN ray.services.get_node_ip_address = lambda: '127.0.0.1' # We can only set the num_cpus if running on a single-node if iconfig.implementation.ip_head is None and iconfig.implementation.redis_password is None: rayargs['num_cpus'] = n_cores ray.init(**rayargs) atexit.register(ray.shutdown) n_ray_cores = ray.available_resources()["CPU"] n_cores = min(n_ray_cores, n_input_lines) logging.info( 'Beginning empirical line inversions using {} cores'.format(n_cores)) # Break data into sections line_sections = np.linspace(0, n_input_lines, num=int(n_cores + 1), dtype=int) start_time = time.time() # Run the pool (or run serially) results = [] for l in range(len(line_sections) - 1): args = (line_sections[l], line_sections[l + 1], reference_radiance_file, reference_reflectance_file, reference_uncertainty_file, reference_locations_file, input_radiance_file, input_locations_file, segmentation_file, isofit_config, output_reflectance_file, output_uncertainty_file, radiance_factors, nneighbors, nodata_value, level, logfile) results.append(_run_chunk.remote(*args)) _ = ray.get(results) total_time = time.time() - start_time logging.info( 'Parallel empirical line inversions complete. {} s total, {} spectra/s, {} spectra/s/core' .format(total_time, line_sections[-1] * n_input_samples / total_time, line_sections[-1] * n_input_samples / total_time / n_cores))
def segment(spectra, flag, npca, segsize, nchunk): """.""" in_file = spectra[0] if len(spectra) > 1: lbl_file = spectra[1] else: lbl_file = spectra + '_lbl' # Open input data, get dimensions in_img = envi.open(in_file + '.hdr', in_file) meta = in_img.metadata nl, nb, ns = [int(meta[n]) for n in ('lines', 'bands', 'samples')] img_mm = in_img.open_memmap(interleave='source', writable=False) if meta['interleave'] != 'bil': raise ValueError('I need BIL interleave.') # Iterate through image "chunks," segmenting as we go next_label = 1 all_labels = s.zeros((nl, ns)) for lstart in s.arange(0, nl, nchunk): del img_mm print(lstart) # Extract data lend = min(lstart + nchunk, nl) img_mm = in_img.open_memmap(interleave='source', writable=False) x = s.array(img_mm[lstart:lend, :, :]).transpose((0, 2, 1)) nc = x.shape[0] x = x.reshape((nc * ns, nb)) # Excluding bad locations, calculate top PCA coefficients use = s.all(abs(x - flag) > 1e-6, axis=1) mu = x[use, :].mean(axis=0) C = s.cov(x[use, :], rowvar=False) [v, d] = eigh(C) # Determine segmentation compactness scaling based on eigenvalues cmpct = norm(s.sqrt(v[-npca:])) # Project, redimension as an image with "npca" channels, and segment x_pca = (x - mu) @ d[:, -npca:] x_pca[use < 1, :] = 0.0 x_pca = x_pca.reshape([nc, ns, npca]) valid = use.reshape([nc, ns, 1]) seg_in_chunk = int(sum(use) / float(segsize)) labels = slic(x_pca, n_segments=seg_in_chunk, compactness=cmpct, max_iter=10, sigma=0, multichannel=True, enforce_connectivity=True, min_size_factor=0.5, max_size_factor=3) # Reindex the subscene labels and place them into the larger scene labels = labels.reshape([nc * ns]) labels[s.logical_not(use)] = 0 labels[use] = labels[use] + next_label next_label = max(labels) + 1 labels = labels.reshape([nc, ns]) all_labels[lstart:lend, :] = labels # Reindex labels_sorted = s.sort(s.unique(all_labels)) lbl = s.zeros((nl, ns)) for i, val in enumerate(labels_sorted): lbl[all_labels == val] = i # Final file I/O del img_mm lbl_meta = { "samples": str(ns), "lines": str(nl), "bands": "1", "header offset": "0", "file type": "ENVI Standard", "data type": "4", "interleave": "bil" } lbl_img = envi.create_image(lbl_file + '.hdr', lbl_meta, ext='', force=True) lbl_mm = lbl_img.open_memmap(interleave='source', writable=True) lbl_mm[:, :] = s.array(lbl, dtype=s.float32).reshape((nl, 1, ns)) del lbl_mm
def extractions(inputfile, labels, output, chunksize, flag, n_cores: int = 1, ray_address: str = None, ray_redis_password: str = None, ray_temp_dir: str = None, ray_ip_head = None, logfile: str = None, loglevel: str = 'INFO'): """...""" in_file = inputfile lbl_file = labels out_file = output nchunk = chunksize dtm = { '4': np.float32, '5': np.float64 } # Open input data, get dimensions in_img = envi.open(envi_header(in_file), in_file) meta = in_img.metadata nl, nb, ns = [int(meta[n]) for n in ('lines', 'bands', 'samples')] img_mm = in_img.open_memmap(interleave='bip', writable=False) lbl_img = envi.open(envi_header(lbl_file), lbl_file) labels = lbl_img.read_band(0) un_labels = np.unique(labels).tolist() if 0 not in un_labels: un_labels.insert(0,0) nout = len(un_labels) # Start up a ray instance for parallel work rayargs = {'ignore_reinit_error': True, 'local_mode': n_cores == 1, "address": ray_address, '_temp_dir': ray_temp_dir, '_redis_password': ray_redis_password} # We can only set the num_cpus if running on a single-node if ray_ip_head is None and ray_redis_password is None: rayargs['num_cpus'] = n_cores ray.init(**rayargs) atexit.register(ray.shutdown) labelid = ray.put(labels) jobs = [] for lstart in np.arange(0, nl, nchunk): lend = min(lstart+nchunk, nl) jobs.append(extract_chunk.remote(lstart, lend, in_file, labelid, flag, logfile=logfile, loglevel=loglevel)) # Collect results rreturn = [ray.get(jid) for jid in jobs] ## Iterate through image "chunks," segmenting as we go out = np.zeros((nout, nb, 1)) for idx, ret in rreturn: if ret is not None: out[idx, :, 0] = ret del rreturn ray.shutdown() meta["lines"] = str(nout) meta["bands"] = str(nb) meta["samples"] = '1' meta["interleave"] = "bil" out_img = envi.create_image(envi_header(out_file), metadata=meta, ext='', force=True) del out_img if dtm[meta['data type']] == np.float32: type = 'float32' else: type = 'float64' write_bil_chunk(out, out_file, 0, out.shape, dtype=type)
def segment(spectra: tuple, nodata_value: float, npca: int, segsize: int, nchunk: int, n_cores: int = 1, ray_address: str = None, ray_redis_password: str = None, ray_temp_dir=None, ray_ip_head=None, logfile=None, loglevel='INFO'): """ Segment an image using SLIC on a PCA. Args: spectra: tuple of filepaths of image to segment and (optionally) output label file nodata_value: data to ignore in radiance image npca: number of pca components to use segsize: mean segmentation size nchunk: size of each image chunk n_cores: number of cores to use ray_address: ray address to connect to (for multinode implementation) ray_redis_password: ray password to use (for multinode implementation) ray_temp_dir: ray temp directory to reference ray_ip_head: ray ip head to reference (for multinode use) logfile: logging file to output to loglevel: logging level to use """ logging.basicConfig(format='%(levelname)s:%(message)s', level=loglevel, filename=logfile) in_file = spectra[0] if len(spectra) > 1 and type(spectra) is tuple: lbl_file = spectra[1] else: lbl_file = spectra + '_lbl' # Open input data, get dimensions in_img = envi.open(envi_header(in_file), in_file) meta = in_img.metadata nl, nb, ns = [int(meta[n]) for n in ('lines', 'bands', 'samples')] # Start up a ray instance for parallel work rayargs = { 'ignore_reinit_error': True, 'local_mode': n_cores == 1, "address": ray_address, '_temp_dir': ray_temp_dir, "_redis_password": ray_redis_password } # We can only set the num_cpus if running on a single-node if ray_ip_head is None and ray_redis_password is None: rayargs['num_cpus'] = n_cores ray.init(**rayargs) atexit.register(ray.shutdown) # Iterate through image "chunks," segmenting as we go all_labels = np.zeros((nl, ns), dtype=np.int64) jobs = [] # Enforce a minimum chunk size to prevent singularities downstream # This could eventually be made a user-tunable parameter but this # value should work in all cases min_lines_per_chunk = 10 for lstart in np.arange(0, nl - min_lines_per_chunk, nchunk): # Extend any chunk that falls within a small margin of the # end of the flightline lend = min(lstart + nchunk, nl) if lend > (nl - min_lines_per_chunk): lend = nl # Extract data jobs.append( segment_chunk.remote(lstart, lend, in_file, nodata_value, npca, segsize, logfile=logfile, loglevel=loglevel)) # Collect results, making sure each chunk is distinct, and enforce an order next_label = 1 rreturn = [ray.get(jid) for jid in jobs] for lstart, lend, ret in rreturn: if ret is not None: logging.debug(f'Collecting chunk: {lstart}') chunk_label = ret.copy() unique_chunk_labels = np.unique(chunk_label[chunk_label != 0]) ordered_chunk_labels = np.zeros(chunk_label.shape) for lbl in unique_chunk_labels: ordered_chunk_labels[chunk_label == lbl] = next_label next_label += 1 all_labels[lstart:lend, ...] = ordered_chunk_labels del rreturn ray.shutdown() # Final file I/O logging.debug('Writing output') lbl_meta = { "samples": str(ns), "lines": str(nl), "bands": "1", "header offset": "0", "file type": "ENVI Standard", "data type": "4", "interleave": "bil" } lbl_img = envi.create_image(envi_header(lbl_file), lbl_meta, ext='', force=True) lbl_mm = lbl_img.open_memmap(interleave='source', writable=True) lbl_mm[:, :] = np.array(all_labels, dtype=np.float32).reshape((nl, 1, ns)) del lbl_mm
def empirical_line(reference_radiance, reference_reflectance, reference_uncertainty, reference_locations, hashfile, input_radiance, input_locations, output_reflectance, output_uncertainty, nneighbors=15, flag=-9999.0, skip=0, level='INFO', radiance_factors=None, isofit_config=None): """...""" def plot_example(xv, yv, b): """Plot for debugging purposes.""" matplotlib.rcParams['font.family'] = "serif" matplotlib.rcParams['font.sans-serif'] = "Times" matplotlib.rcParams["legend.edgecolor"] = "None" matplotlib.rcParams["axes.spines.top"] = False matplotlib.rcParams["axes.spines.bottom"] = True matplotlib.rcParams["axes.spines.left"] = True matplotlib.rcParams["axes.spines.right"] = False matplotlib.rcParams['axes.grid'] = True matplotlib.rcParams['axes.grid.axis'] = 'both' matplotlib.rcParams['axes.grid.which'] = 'major' matplotlib.rcParams['legend.edgecolor'] = '1.0' plt.plot(xv[:, 113], yv[:, 113], 'ko') plt.plot(xv[:, 113], xv[:, 113] * b[113, 1] + b[113, 0], 'k') #plt.plot(x[113], x[113]*b[113, 1] + b[113, 0], 'ro') plt.grid(True) plt.xlabel('Radiance, $\mu{W }nm^{-1} sr^{-1} cm^{-2}$') plt.ylabel('Reflectance') plt.show(block=True) plt.savefig('empirical_line.pdf') eps = 1e-6 k = nneighbors loglevel = level # Open input data, get dimensions logging.basicConfig(format='%(message)s', level=loglevel) ref_rdn_file = reference_radiance ref_rfl_file = reference_reflectance ref_unc_file = reference_uncertainty ref_loc_file = reference_locations inp_hash_file = hashfile inp_rdn_file = input_radiance inp_loc_file = input_locations out_rfl_file = output_reflectance out_unc_file = output_uncertainty # Load reference set radiance ref_rdn_img = envi.open(ref_rdn_file + '.hdr', ref_rdn_file) meta = ref_rdn_img.metadata nref, nb, sref = [int(meta[n]) for n in ('lines', 'bands', 'samples')] if sref != 1: raise IndexError("Reference data should be a single-column list") ref_rdn_mm = ref_rdn_img.open_memmap(interleave='source', writable=False) ref_rdn = s.array(ref_rdn_mm[:, :, :]).reshape((nref, nb)) # Load reference set reflectance ref_rfl_img = envi.open(ref_rfl_file + '.hdr', ref_rfl_file) meta = ref_rfl_img.metadata nrefr, nbr, srefr = [int(meta[n]) for n in ('lines', 'bands', 'samples')] if nrefr != nref or nbr != nb or srefr != sref: raise IndexError("Reference file dimension mismatch (reflectance)") ref_rfl_mm = ref_rfl_img.open_memmap(interleave='source', writable=False) ref_rfl = s.array(ref_rfl_mm[:, :, :]).reshape((nref, nb)) # Load reference set uncertainty, assuming reflectance uncertainty is # recoreded in the first nbr channels of data ref_unc_img = envi.open(ref_unc_file + '.hdr', ref_unc_file) meta = ref_unc_img.metadata nrefu, ns, srefu = [int(meta[n]) for n in ('lines', 'bands', 'samples')] if nrefu != nref or ns < nb or srefu != sref: raise IndexError("Reference file dimension mismatch (uncertainty)") ref_unc_mm = ref_unc_img.open_memmap(interleave='source', writable=False) ref_unc = s.array(ref_unc_mm[:, :, :]).reshape((nref, ns)) ref_unc = ref_unc[:, :nbr].reshape((nref, nbr)) # Load reference set locations ref_loc_img = envi.open(ref_loc_file + '.hdr', ref_loc_file) meta = ref_loc_img.metadata nrefl, lb, ls = [int(meta[n]) for n in ('lines', 'bands', 'samples')] if nrefl != nref or lb != 3: raise IndexError("Reference file dimension mismatch (locations)") ref_loc_mm = ref_loc_img.open_memmap(interleave='source', writable=False) ref_loc = s.array(ref_loc_mm[:, :, :]).reshape((nref, lb)) # Prepare radiance adjustment if radiance_factors is None: rdn_factors = s.ones(nb, ) else: rdn_factors = s.loadtxt(radiance_factors) # Prepare instrument model, if available if isofit_config is not None: config = load_config(isofit_config) instrument = Instrument(config['forward_model']['instrument']) logging.info('Loading instrument') else: instrument = None # Assume (heuristically) that, for distance purposes, 1 m vertically is # comparable to 10 m horizontally, and that there are 100 km per latitude # degree. This is all approximate of course. Elevation appears in the # Third element, and the first two are latitude/longitude coordinates loc_scaling = s.array([1e5, 1e5, 0.1]) scaled_ref_loc = ref_loc * loc_scaling tree = KDTree(scaled_ref_loc) inp_rdn_img = envi.open(inp_rdn_file + '.hdr', inp_rdn_file) inp_rdn_meta = inp_rdn_img.metadata nl, nb, ns = [int(inp_rdn_meta[n]) for n in ('lines', 'bands', 'samples')] if nb != nbr: msg = 'Number of channels mismatch: input (%i) vs. reference (%i)' raise IndexError(msg % (nbr, nb)) inp_rdn_mm = inp_rdn_img.open_memmap(interleave='source', writable=False) inp_loc_img = envi.open(inp_loc_file + '.hdr', inp_loc_file) inp_loc_meta = inp_loc_img.metadata nll, nlb, nls = [ int(inp_loc_meta[n]) for n in ('lines', 'bands', 'samples') ] if nll != nl or nlb != 3 or nls != ns: raise IndexError('Input location dimension mismatch') inp_loc_mm = inp_loc_img.open_memmap(interleave='source', writable=False) inp_loc = s.array(inp_loc_mm).reshape((nl, nlb, ns)) if inp_hash_file: inp_hash_img = envi.open(inp_hash_file + '.hdr', inp_hash_file) hash_img = inp_hash_img.read_band(0) else: hash_img = None out_rfl_img = envi.create_image(out_rfl_file + '.hdr', ext='', metadata=inp_rdn_img.metadata, force=True) out_rfl_mm = out_rfl_img.open_memmap(interleave='source', writable=True) out_unc_img = envi.create_image(out_unc_file + '.hdr', ext='', metadata=inp_rdn_img.metadata, force=True) out_unc_mm = out_unc_img.open_memmap(interleave='source', writable=True) # Iterate through image hash_table = {} for row in s.arange(nl): del inp_loc_mm del inp_rdn_mm del out_rfl_mm del out_unc_mm # Extract data inp_rdn_mm = inp_rdn_img.open_memmap(interleave='source', writable=False) inp_rdn = s.array(inp_rdn_mm[row, :, :]) if inp_rdn_meta['interleave'] == 'bil': inp_rdn = inp_rdn.transpose((1, 0)) inp_rdn = inp_rdn * rdn_factors inp_loc_mm = inp_loc_img.open_memmap(interleave='source', writable=False) inp_loc = s.array(inp_loc_mm[row, :, :]) if inp_loc_meta['interleave'] == 'bil': inp_loc = inp_loc.transpose((1, 0)) out_rfl = s.zeros(inp_rdn.shape) out_unc = s.zeros(inp_rdn.shape) nspectra, start = 0, time.time() for col in s.arange(ns): x = inp_rdn[col, :] if s.all(abs(x - flag) < eps): out_rfl[col, :] = flag out_unc[col, :] = flag continue bhat = None if hash_img is not None: hash_idx = hash_img[row, col] if hash_idx in hash_table: bhat, bmarg, bcov = hash_table[hash_idx] else: loc = ref_loc[ s.array(hash_idx, dtype=int), :] * loc_scaling else: loc = inp_loc[col, :] * loc_scaling if bhat is None: dists, nn = tree.query(loc, k) xv = ref_rdn[nn, :] yv = ref_rfl[nn, :] uv = ref_unc[nn, :] bhat = s.zeros((nb, 2)) bmarg = s.zeros((nb, 2)) bcov = s.zeros((nb, 2, 2)) for i in s.arange(nb): use = yv[:, i] > 0 n = sum(use) X = s.concatenate((s.ones((n, 1)), xv[use, i:i + 1]), axis=1) W = s.diag(s.ones(n)) # /uv[use, i]) y = yv[use, i:i + 1] bhat[i, :] = (inv(X.T @ W @ X) @ X.T @ W @ y).T bcov[i, :, :] = inv(X.T @ W @ X) bmarg[i, :] = s.diag(bcov[i, :, :]) if (hash_img is not None) and not (hash_idx in hash_table): hash_table[hash_idx] = bhat, bmarg, bcov A = s.array((s.ones(nb), x)) out_rfl[col, :] = (s.multiply(bhat.T, A).sum(axis=0)) # Calculate uncertainties. Sy approximation rather than Seps for # speed, for now... but we do take into account instrument # radiometric uncertainties if instrument is None: out_unc[col, :] = s.sqrt(s.multiply(bmarg.T, A).sum(axis=0)) else: Sy = instrument.Sy(x, geom=None) calunc = instrument.bval[:instrument.n_chan] out_unc[col, :] = s.sqrt(s.diag(Sy) + pow(calunc * x, 2)) * bhat[:, 1] if loglevel == 'DEBUG': plot_example(xv, yv, bhat, x, out_rfl[col, :]) nspectra = nspectra + 1 elapsed = float(time.time() - start) logging.info('row %i/%i, %5.1f spectra per second' % (row, nl, float(nspectra) / elapsed)) out_rfl_mm = out_rfl_img.open_memmap(interleave='source', writable=True) if inp_rdn_meta['interleave'] == 'bil': out_rfl = out_rfl.transpose((1, 0)) out_rfl_mm[row, :, :] = out_rfl out_unc_mm = out_unc_img.open_memmap(interleave='source', writable=True) if inp_rdn_meta['interleave'] == 'bil': out_unc = out_unc.transpose((1, 0)) out_unc_mm[row, :, :] = out_unc
def main(in_file): # Set up input and output file paths in_hdr = find_header(in_file) ########################## ## MODIFIY THIS TO GET A BETTER FILENAME ########################## out_glint_file = in_file.split('_', 1)[0] + '_glint' out_glint_hdr = in_file.split('_', 1)[0] + '_glint.hdr' img = envi.open(in_hdr, in_file) inmm = img.open_memmap(interleave='source', writable=False) wl = s.array([float(w) for w in img.metadata['wavelength']]) if (wl[0] < 100): wl = wl * 1000 fwhm = s.array([float(w) for w in img.metadata['fwhm']]) if (wl[0] < 100): fwhm = fwhm * 1000 # set up metadata and constants # make output glint file and open memmap nl = int(img.metadata['lines']) metadata_glint = img.metadata.copy() metadata_glint['bands'] = u'1' metadata_glint['data type'] = u'4' metadata_glint['band names'] = ['Glint at 900nm'] metadata_glint['interleave'] = 'bsq' metadata_glint[ 'description'] = ' make_glint.py from input ATREM reflectance ' try: del metadata_glint['wavelength'] del metadata_glint['wavelength units'] del metadata_glint['fwhm'] del metadata_glint['raw starting band'] del metadata_glint['raw starting sample'] del metadata_glint['raw starting line'] del metadata_glint['line averaging'] except: pass out_glint = envi.create_image(out_glint_hdr, metadata_glint, ext='', force=True) outmm_glint = out_glint.open_memmap(interleave='source', writable=True) # iterate over rows start, fin = 0, nl for i in range(start, fin): Rw = s.array(inmm[i, :, :]) if img.metadata['interleave'] == 'bil': Rw = Rw.T ## if the ENVI data type is unsigned 16-bit integer, just change it to be signed ## so that numpy doesn't interpret them as insanely high numbers (e.g., 65535) if int(img.metadata['data type']) == 12: Rw.dtype = 'int16' if int(img.metadata['data type']) != 4: Rw = Rw / s.float32(10000) # iterate over columns colstart, colfin = 0, img.ncols glint_frame = s.zeros((outmm_glint.shape[2], 1)) for col in range(colstart, colfin): # check for land and bad data flags ## if Rw[col,s.argmin(abs(wl-1000))] > 0.05 or all(Rw[col,:] <= 0): if Rw[col, s.argmin(abs(wl - 1000))] > 0.10 or all( Rw[col, :] <= 0): continue # convert to Rrs Rrs_raw = Rw[col, :] / s.pi # subtract glint b900 = s.argmin(abs(wl - 900.0)) glint = max(0.0001, s.median(Rrs_raw[(b900 - 2):(b900 + 3)])) if all(Rrs_raw < 0): continue # out of bounds data # write output files glint_state = glint glint_frame[col, 0] = glint_state outmm_glint[0, i, :] = glint_frame.reshape(img.ncols) if ((i % 500) == 0): print 'line ' + str(i + 1) del outmm_glint, out_glint, inmm, img
def main(): # Parse command line description = 'Spectroscopic Surface & Atmosphere Fitting' parser = argparse.ArgumentParser() parser.add_argument('config_file') parser.add_argument('--level', default='INFO') parser.add_argument('--ip_head', default=None, help='ray-specific argument') parser.add_argument('--redis_password', default=None, help='ray-specific argument') parser.add_argument('--ray_temp_dir', default=None, help='ray-specific argument') parser.add_argument( '--n_cores', type=int, default=-1, help="number of cores to run on. -1 for all, 1 for debug mode") args = parser.parse_args() logging.basicConfig(format='%(message)s', level=args.level) # Load a parallel Pool rayargs = { 'address': args.ip_head, 'redis_password': args.redis_password, 'local_mode': args.n_cores == 1 } if args.n_cores != -1: rayargs['num_cpus'] = args.n_cores if args.ray_temp_dir is not None: rayargs['temp_dir'] = args.ray_temp_dir ray.init(**rayargs) # Load the configuration file. config = json.load(open(args.config_file, 'r')) logging.info('Loading library') lib = Library(config['library']) # Get image and wavelengths logging.info('Opening input data') reflectance_input_header = str(config['input_reflectance'] + '.hdr') uncertainty_input_header = str(config['input_uncertainty'] + '.hdr') depth_output_header = str(config['output_depths'] + '.hdr') posterior_output_header = str(config['output_posterior'] + '.hdr') likelihood_output_header = str(config['output_likelihood'] + '.hdr') corr_output_header = str(config['output_corr'] + '.hdr') reflectance_ds = envi.open(reflectance_input_header) meta = reflectance_ds.metadata.copy() uncertainty_ds = envi.open(uncertainty_input_header) # Now that the input images are available, resample wavelengths if 'wavelength' in meta: wl = np.array([float(w) for w in meta['wavelength']]) if all(wl < 100): wl = wl * 1000.0 else: wl = lib.wl.copy() if 'fwhm' in meta: fwhm = np.array([float(f) for f in meta['fwhm']]) if all(fwhm < 0.1): fwhm = fwhm * 1000.0 else: fwhm = np.ones(wl.shape) * (wl[1] - wl[0]) lib.resample(wl, fwhm) # Create output images meta['bands'] = lib.nchan() if 'wavelength' in meta: del meta['wavelength'] if 'fwhm' in meta: del meta['fwhm'] meta['band names'] = lib.groups() meta['data type'] = 4 meta['interleave'] = 'bip' depth_ds = envi.create_image(depth_output_header, meta, force=True, ext="") posterior_ds = envi.create_image(posterior_output_header, meta, force=True, ext="") likelihood_ds = envi.create_image(likelihood_output_header, meta, force=True, ext="") corr_ds = envi.create_image(corr_output_header, meta, force=True, ext="") ids = [ run_one_row.remote(r, lib, reflectance_input_header, uncertainty_input_header, depth_output_header, posterior_output_header, likelihood_output_header, corr_output_header) for r in range(reflectance_ds.shape[0]) ] ret = [ray.get(id) for id in ids]
inmm = img.open_memmap(interleave='source', writable=False) wl = s.array([float(w) for w in img.metadata['wavelength']]) if (wl[0] < 100): wl = wl * 1000 fwhm = s.array([float(w) for w in img.metadata['fwhm']]) # set up metadata and constants const = constants(wl, args.mode, scenario, args.root_dir+'/data/') # make output Rb file and open memmap metadata = img.metadata.copy() metadata['bands'] = '%i' % sum(const.use_out) metadata['interleave'] = 'bil' metadata['wavelength'] = ['%7.2f'%w for w in wl[const.use_out]] metadata['data type'] = '2' out = envi.create_image(out_hdr, metadata, ext='',force=True) outmm = out.open_memmap(interleave='source', writable=True) for i in range(start, fin): # Flush cache print 'line %i/%i' % (i,nl) if args.depth_file is not None: del bathy bathy= spectral.io.envi.open(args.depth_file+'.hdr', args.depth_file) bathymm = bathy.open_memmap(interleave='source', writable=False) bath = s.array(bathymm[i,:,:]) if bathy.metadata['interleave'] == 'bil': bath = bath.T
def extractions(inputfile, labels, output, chunksize, flag): """...""" in_file = inputfile lbl_file = labels out_file = output nchunk = chunksize dtm = {'4': np.float32, '5': np.float64} # Open input data, get dimensions in_img = envi.open(in_file + '.hdr', in_file) meta = in_img.metadata nl, nb, ns = [int(meta[n]) for n in ('lines', 'bands', 'samples')] img_mm = in_img.open_memmap(interleave='bip', writable=False) lbl_img = envi.open(lbl_file + '.hdr', lbl_file) labels = lbl_img.read_band(0) nout = len(np.unique(labels)) # Iterate through image "chunks," segmenting as we go out = np.zeros((nout, nb)) counts = np.zeros((nout)) for lstart in np.arange(0, nl, nchunk): del img_mm img_mm = in_img.open_memmap(interleave='bip', writable=False) # Which labels will we extract? ignore zero index lend = min(lstart + nchunk, nl) active = np.unique(labels[lstart:lend, :]) active = active[active >= 1] # Handle labels extending outside our chunk by expanding margins active_area = np.zeros(labels.shape) lstart_adjust, lend_adjust = lstart, lend for i in active: active_area[labels == i] = True active_locs = np.where(active_area) lstart_adjust = min(active_locs[0]) lend_adjust = max(active_locs[0]) + 1 chunk_inp = np.array(img_mm[lstart_adjust:lend_adjust, :, :]) chunk_lbl = np.array(labels[lstart_adjust:lend_adjust, :]) for i in active: idx = int(i) out[idx, :] = 0 locs = np.where(chunk_lbl == i) for row, col in zip(locs[0], locs[1]): out[idx, :] = out[idx, :] + np.squeeze(chunk_inp[row, col, :]) counts[idx] = len(locs[0]) out = np.array((out.T / counts[np.newaxis, :]).T, dtype=np.float32) out[np.logical_not(np.isfinite(out))] = flag meta["lines"] = str(nout) meta["bands"] = str(nb) meta["samples"] = '1' meta["interleave"] = "bil" out_img = envi.create_image(out_file + '.hdr', metadata=meta, ext='', force=True) out_mm = np.memmap(out_file, dtype=dtm[meta['data type']], mode='w+', shape=(nout, 1, nb)) if dtm[meta['data type']] == np.float32: out_mm[:, 0, :] = np.array(out, np.float32) else: out_mm[:, 0, :] = np.array(out, np.float64)
def createimg(hdrf,metadata,ext='',force=True): from spectral.io.envi import create_image return create_image(hdrf, metadata, ext=ext, force=force)
def __init__(self, fname, write=False, n_rows=None, n_cols=None, n_bands=None, interleave=None, dtype=np.float32, wavelengths=None, fwhm=None, band_names=None, bad_bands='[]', zrange='{0.0, 1.0}', flag=-9999.0, ztitles='{Wavelength (nm), Magnitude}', map_info='{}'): """.""" self.frames = OrderedDict() self.write = write self.fname = os.path.abspath(fname) self.wl = wavelengths self.band_names = band_names self.fwhm = fwhm self.flag = flag self.n_rows = n_rows self.n_cols = n_cols self.n_bands = n_bands if self.fname.endswith('.txt'): # The .txt suffix implies a space-separated ASCII text file of # one or more data columns. This is cheap to load and store, so # we do not defer read/write operations. logging.debug('Inferred ASCII file format for %s' % self.fname) self.format = 'ASCII' if not self.write: self.data, self.wl = load_spectrum(self.fname) self.n_rows, self.n_cols, self.map_info = 1, 1, '{}' if self.wl is not None: self.n_bands = len(self.wl) else: self.n_bands = None self.meta = {} elif self.fname.endswith('.mat'): # The .mat suffix implies a matlab-style file, i.e. a dictionary # of 2D arrays and other matlab-like objects. This is typically # only used for specific output products associated with single # spectrum retrievals; there is no read option. logging.debug('Inferred MATLAB file format for %s' % self.fname) self.format = 'MATLAB' if not self.write: logging.error('Unsupported MATLAB file in input block') raise IOError('MATLAB format in input block not supported') else: # Otherwise we assume it is an ENVI-format file, which is # basically just a binary data cube with a detached human- # readable ASCII header describing dimensions, interleave, and # metadata. We buffer this data in self.frames, reading and # writing individual rows of the cube on-demand. logging.debug('Inferred ENVI file format for %s' % self.fname) self.format = 'ENVI' if not self.write: # If we are an input file, the header must preexist. if not os.path.exists(self.fname+'.hdr'): logging.error('Could not find %s' % (self.fname+'.hdr')) raise IOError('Could not find %s' % (self.fname+'.hdr')) # open file and copy metadata self.file = envi.open(self.fname + '.hdr', fname) self.meta = self.file.metadata.copy() self.n_rows = int(self.meta['lines']) self.n_cols = int(self.meta['samples']) self.n_bands = int(self.meta['bands']) if 'data ignore value' in self.meta: self.flag = float(self.meta['data ignore value']) else: self.flag = -9999.0 else: # If we are an output file, we may have to build the header # from scratch. Hopefully the caller has supplied the # necessary metadata details. meta = { 'lines': n_rows, 'samples': n_cols, 'bands': n_bands, 'byte order': 0, 'header offset': 0, 'map info': map_info, 'file_type': 'ENVI Standard', 'sensor type': 'unknown', 'interleave': interleave, 'data type': typemap[dtype], 'wavelength units': 'nm', 'z plot range': zrange, 'z plot titles': ztitles, 'fwhm': fwhm, 'bbl': bad_bands, 'band names': band_names, 'wavelength': self.wl } for k, v in meta.items(): if v is None: logging.error('Must specify %s' % (k)) raise IOError('Must specify %s' % (k)) if os.path.isfile(fname+'.hdr') is False: self.file = envi.create_image(fname+'.hdr', meta, ext='', force=True) else: self.file = envi.open(fname+'.hdr') self.open_map_with_retries()
def empirical_line(reference_radiance_file: str, reference_reflectance_file: str, reference_uncertainty_file: str, reference_locations_file: str, segmentation_file: str, input_radiance_file: str, input_locations_file: str, output_reflectance_file: str, output_uncertainty_file: str, nneighbors: int = 15, nodata_value: float = -9999.0, level: str = 'INFO', radiance_factors: np.array = None, isofit_config: dict = None, n_cores: int = -1) -> None: """ Perform an empirical line interpolation for reflectance and uncertainty extrapolation Args: reference_radiance_file: source file for radiance (interpolation built from this) reference_reflectance_file: source file for reflectance (interpolation built from this) reference_uncertainty_file: source file for uncertainty (interpolation built from this) reference_locations_file: source file for file locations (lon, lat, elev), (interpolation built from this) segmentation_file: input file noting the per-pixel segmentation used input_radiance_file: input radiance file (interpolate over this) input_locations_file: input location file (interpolate over this) output_reflectance_file: location to write output reflectance to output_uncertainty_file: location to write output uncertainty to nneighbors: number of neighbors to use for interpolation nodata_value: nodata value of input and output level: logging level radiance_factors: radiance adjustment factors isofit_config: dictionary-stype isofit configuration n_cores: number of cores to run on Returns: None """ loglevel = level logging.basicConfig(format='%(message)s', level=loglevel) # Open input data to check that band formatting is correct # Load reference set radiance reference_radiance_img = envi.open(reference_radiance_file + '.hdr', reference_radiance_file) n_reference_lines, n_radiance_bands, n_reference_columns = [ int(reference_radiance_img.metadata[n]) for n in ('lines', 'bands', 'samples') ] if n_reference_columns != 1: raise IndexError("Reference data should be a single-column list") # Load reference set reflectance reference_reflectance_img = envi.open(reference_reflectance_file + '.hdr', reference_reflectance_file) nrefr, nbr, srefr = [ int(reference_reflectance_img.metadata[n]) for n in ('lines', 'bands', 'samples') ] if nrefr != n_reference_lines or nbr != n_radiance_bands or srefr != n_reference_columns: raise IndexError("Reference file dimension mismatch (reflectance)") # Load reference set uncertainty, assuming reflectance uncertainty is # recoreded in the first n_radiance_bands channels of data reference_uncertainty_img = envi.open(reference_uncertainty_file + '.hdr', reference_uncertainty_file) nrefu, ns, srefu = [ int(reference_uncertainty_img.metadata[n]) for n in ('lines', 'bands', 'samples') ] if nrefu != n_reference_lines or ns < n_radiance_bands or srefu != n_reference_columns: raise IndexError("Reference file dimension mismatch (uncertainty)") # Load reference set locations reference_locations_img = envi.open(reference_locations_file + '.hdr', reference_locations_file) nrefl, lb, ls = [ int(reference_locations_img.metadata[n]) for n in ('lines', 'bands', 'samples') ] if nrefl != n_reference_lines or lb != 3: raise IndexError("Reference file dimension mismatch (locations)") input_radiance_img = envi.open(input_radiance_file + '.hdr', input_radiance_file) n_input_lines, n_input_bands, n_input_samples = [ int(input_radiance_img.metadata[n]) for n in ('lines', 'bands', 'samples') ] if n_radiance_bands != n_input_bands: msg = 'Number of channels mismatch: input (%i) vs. reference (%i)' raise IndexError(msg % (nbr, n_radiance_bands)) input_locations_img = envi.open(input_locations_file + '.hdr', input_locations_file) nll, nlb, nls = [ int(input_locations_img.metadata[n]) for n in ('lines', 'bands', 'samples') ] if nll != n_input_lines or nlb != 3 or nls != n_input_samples: raise IndexError('Input location dimension mismatch') # Create output files output_metadata = input_radiance_img.metadata output_metadata['interleave'] = 'bil' output_reflectance_img = envi.create_image(output_reflectance_file + '.hdr', ext='', metadata=output_metadata, force=True) output_uncertainty_img = envi.create_image(output_uncertainty_file + '.hdr', ext='', metadata=output_metadata, force=True) # Now cleanup inputs and outputs, we'll write dynamically above del output_reflectance_img, output_uncertainty_img del reference_reflectance_img, reference_uncertainty_img, reference_locations_img, input_radiance_img, input_locations_img # Determine the number of cores to use if n_cores == -1: n_cores = multiprocessing.cpu_count() n_cores = min(n_cores, n_input_lines) # Break data into sections line_sections = np.linspace(0, n_input_lines, num=n_cores + 1, dtype=int) # Set up our pool pool = multiprocessing.Pool(processes=n_cores) start_time = time.time() logging.info( 'Beginning empirical line inversions using {} cores'.format(n_cores)) # Run the pool (or run serially) results = [] for l in range(len(line_sections) - 1): args = ( line_sections[l], line_sections[l + 1], reference_radiance_file, reference_reflectance_file, reference_uncertainty_file, reference_locations_file, input_radiance_file, input_locations_file, segmentation_file, isofit_config, output_reflectance_file, output_uncertainty_file, radiance_factors, nneighbors, nodata_value, ) if n_cores != 1: results.append(pool.apply_async(_run_chunk, args)) else: _run_chunk(*args) pool.close() pool.join() total_time = time.time() - start_time logging.info( 'Parallel empirical line inversions complete. {} s total, {} spectra/s, {} spectra/s/core' .format(total_time, line_sections[-1] * n_input_samples / total_time, line_sections[-1] * n_input_samples / total_time / n_cores))