def slope_aspect(elevation, temp_dir): ''' Use GDAL to calculte slope and aspect ''' dem_dict = envi_header_dict() dem_dict['lines'] = elevation.shape[0] dem_dict['samples'] = elevation.shape[1] dem_dict['bands'] = 1 dem_dict['interleave'] = 'bsq' dem_dict['data type'] = 4 dem_file = '%stemp_dem_clip' % temp_dir writer = WriteENVI(dem_file, dem_dict) writer.write_band(elevation, 0) slope_file = '%s_slope' % temp_dir aspect_file = '%s_aspect' % temp_dir logging.info('Calculating slope') os.system('gdaldem slope -of ENVI %s %s' % (dem_file, slope_file)) logging.info('Calculating aspect') os.system('gdaldem aspect -f ENVI %s %s' % (dem_file, aspect_file)) asp_obj = ht.HyTools() asp_obj.read_file(aspect_file, 'envi') aspect = asp_obj.get_band(0) slp_obj = ht.HyTools() slp_obj.read_file(slope_file, 'envi') slope = slp_obj.get_band(0) return slope, aspect
def gen_wavelength_file(rad_file, out_dir): '''Export radiance image wavelengths and FWHM to file ''' radiance = ht.HyTools() radiance.read_file(rad_file, 'envi') wave_fit = pd.DataFrame(index=[x for x in range(radiance.bands)]) wave_fit[0] = np.round(radiance.wavelengths / 1000, 5) wave_fit[1] = np.round(radiance.fwhm / 1000, 5) wavelength_file = '%s/wavelength_fit.txt' % out_dir wave_fit.to_csv(wavelength_file, sep='\t', header=False) return wavelength_file
def create_loc_ort(loc_file,glt_file): loc = ht.HyTools() loc.read_file(loc_file,'envi') glt = ht.HyTools() glt.read_file(glt_file,'envi') samples = np.abs(glt.get_band(0)) lines = np.abs(glt.get_band(1)) lon = np.copy(loc.get_band(0)) lat= np.copy(loc.get_band(1)) elv = np.copy(loc.get_band(2)) lon_proj = lon[lines.flatten()-1,samples.flatten()-1] lon_proj = lon_proj.reshape(lines.shape) lat_proj = lat[lines.flatten()-1,samples.flatten()-1] lat_proj = lat_proj.reshape(lines.shape) elv_proj = elv[lines.flatten()-1,samples.flatten()-1] elv_proj = elv_proj.reshape(lines.shape) lon_proj[samples ==0] = -9999 lat_proj[samples ==0] = -9999 elv_proj[samples ==0] = -9999 # Create output file header_dict =glt.get_header() header_dict['bands'] = 3 header_dict['data type'] = 4 header_dict['band names'] = loc.get_header()['band names'] header_dict['description'] = loc.get_header()['description'] writer = WriteENVI(loc_file +'_ort',header_dict) writer.write_band(lon_proj,0) writer.write_band(lat_proj,1) writer.write_band(elv_proj,2) writer.close()
def time_correct(obs_ort_file): obs = ht.HyTools() obs.read_file(obs_ort_file,'envi') utc_time = obs.get_band(9)[obs.mask['no_data']] if (utc_time.max() > 24) | (utc_time.min() < 0): hour = np.ones((obs.lines,obs.columns)) hour += int(obs.base_name[12:14]) hour += int(obs.base_name[14:16])/60 hour += int(obs.base_name[16:18])/3600 hour[~obs.mask['no_data']] = obs.no_data obs.load_data(mode = 'r+') if obs.interleave == 'bip': obs.data[:,:,9] =hour elif obs.interleave == 'bil': obs.data[:,9,:] =hour elif obs.interleave == 'bsq': obs.data[9,:,:] =hour obs.close_data()
def he5_to_envi(l1_zip,out_dir,temp_dir,elev_dir,shift = False, rad_coeff = None, match=False,proj = False,res = 30): ''' This function exports three files: *_rdn* : Merged and optionally shift corrected radiance cube *_obs* : Observables file in the format of JPL obs files: 1. Pathlength (m) 2. To-sensor view azimuth angle (degrees) 3. To-sensor view zenith angle (degrees) 4. To-sun azimuth angle (degrees) 5. To-sun zenith angle (degrees) 6. Phase 7. Slope (Degrees) 8. Aspect (Degrees) 9. Cosine i 10. UTC decimal hours *_loc* : Location file in the following format: 1. Longitude (decimal degrees) 2. Longitude (decimal degrees) 3. Elevation (m) l1(str): L1 zipped radiance data product path out_dir(str): Output directory of ENVI datasets temp_dir(str): Temporary directory for intermediate elev_dir (str): Directory zipped Copernicus elevation tiles or url to AWS Copernicus data ex : 'https://copernicus-dem-30m.s3.amazonaws.com/' shift (bool) : Apply wavelength shift correction surface file rad_coeff (bool) : Apply radiometric correction coefficients file match (bool or string) : Perform landsat image matching, if string path to reference file proj (bool) : Project image to UTM grid res (int) : Resolution of projected image, 30 should be one of its factors (90,120,150.....) ''' base_name = os.path.basename(l1_zip)[16:-4] out_dir = "%s/PRS_%s/" % (out_dir,base_name) if not os.path.isdir(out_dir): os.mkdir(out_dir) logging.basicConfig(filename='%s/PRS_%s.log' % (out_dir,base_name), format='%(asctime)s: %(levelname)s - %(message)s', datefmt='%Y-%m-%d %H:%M:%S', level=logging.NOTSET) temp_dir = '%s/tmpPRS_%s/'% (temp_dir,base_name) if not os.path.isdir(temp_dir): os.mkdir(temp_dir) zip_base =os.path.basename(l1_zip) logging.info('Unzipping %s' % zip_base) with zipfile.ZipFile(l1_zip,'r') as zipped: zipped.extractall(temp_dir) l1_obj = h5py.File('%sPRS_L1_STD_OFFL_%s.he5' % (temp_dir,base_name),'r') if shift: shift_file = importlib.resources.open_binary(data,"PRS_20210409105743_20210409105748_0001_wavelength_shift_surface.npz") shift_obj = np.load(shift_file) shift_surface = shift_obj['shifts'] #interp_kind = shift_obj['interp_kind'] interp_kind='quadratic' coeff_arr = np.ones((996, 230)) if rad_coeff != None: coeff_file = importlib.resources.open_binary(data,"PRS_20210409105743_20210409105748_0001_radcoeff_surface.npz") coeff_obj = np.load(coeff_file) if rad_coeff == 'full': coeff_arr = coeff_obj['coeffs'] elif rad_coeff == 'mean': coeff_arr[:] = coeff_obj['coeffs'].mean(axis=0) elif rad_coeff == 'center': coeff_arr[:] = coeff_obj['coeffs'][498-25:498+25].mean(axis=0) else: print('Unrecognized coeff type') #Define output paths if proj: rdn_file = '%sPRS_%s_rdn' % (temp_dir,base_name) loc_file = '%sPRS_%s_loc' % (temp_dir,base_name) obs_file = '%sPRS_%s_obs' % (temp_dir,base_name) else: rdn_file = '%sPRS_%s_rdn' % (out_dir,base_name) loc_file = '%sPRS_%s_loc' % (out_dir,base_name) obs_file = '%sPRS_%s_obs' % (out_dir,base_name) measurement = 'rdn' logging.info('Exporting radiance data') # Export VNIR to temporary ENVI vnir_data = l1_obj['HDFEOS']["SWATHS"]['PRS_L1_HCO']['Data Fields']['VNIR_Cube'] vnir_waves = l1_obj.attrs.get('List_Cw_Vnir') vnir_fwhm = l1_obj.attrs.get('List_Fwhm_Vnir') rdn_dict = envi_header_dict () rdn_dict['lines']= vnir_data.shape[0] rdn_dict['samples']= vnir_data.shape[2] rdn_dict['bands']= vnir_data.shape[1] rdn_dict['wavelength']= vnir_waves rdn_dict['fwhm']= vnir_fwhm rdn_dict['interleave']= 'bsq' rdn_dict['data type'] = 12 rdn_dict['wavelength units'] = "nanometers" rdn_dict['byte order'] = 0 vnir_temp = '%sPRS_%s_%s_vnir' % (temp_dir,base_name,measurement) writer = WriteENVI(vnir_temp,rdn_dict ) writer.write_chunk(np.moveaxis(vnir_data[:,:,:],1,2), 0,0) # Export SWIR to temporary ENVI swir_data = l1_obj['HDFEOS']["SWATHS"]['PRS_L1_HCO']['Data Fields']['SWIR_Cube'] swir_waves = l1_obj.attrs.get('List_Cw_Swir') swir_fwhm = l1_obj.attrs.get('List_Fwhm_Swir') rdn_dict['lines']= swir_data.shape[0] rdn_dict['samples']= swir_data.shape[2] rdn_dict['bands']= swir_data.shape[1] rdn_dict['wavelength']= swir_waves rdn_dict['fwhm']= swir_fwhm swir_temp = '%sPRS_%s_%s_swir' % (temp_dir,base_name,measurement) writer = WriteENVI(swir_temp,rdn_dict ) writer.write_chunk(np.moveaxis(swir_data[:,:,:],1,2), 0,0) vnir_waves = np.flip(vnir_waves[3:]) #6 swir_waves = np.flip(swir_waves[:-6]) #-3 vnir_fwhm = np.flip(vnir_fwhm[3:]) swir_fwhm = np.flip(swir_fwhm[:-6]) vnir_obj = ht.HyTools() vnir_obj.read_file(vnir_temp, 'envi') swir_obj = ht.HyTools() swir_obj.read_file(swir_temp, 'envi') rdn_dict = envi_header_dict() rdn_dict ['lines']= vnir_obj.lines-4 #Clip edges of array rdn_dict ['samples']=vnir_obj.columns-4 #Clip edges of array rdn_dict ['bands']= len(vnir_waves.tolist() + swir_waves.tolist()) rdn_dict ['wavelength']= vnir_waves.tolist() + swir_waves.tolist() rdn_dict ['fwhm']= vnir_fwhm.tolist() + swir_fwhm.tolist() rdn_dict ['interleave']= 'bil' rdn_dict ['data type'] = 4 rdn_dict ['wavelength units'] = "nanometers" rdn_dict ['byte order'] = 0 rdn_dict ['default bands'] = [int(vnir_obj.wave_to_band(660)), int(vnir_obj.wave_to_band(560)), int(vnir_obj.wave_to_band(460))] writer = WriteENVI(rdn_file,rdn_dict) iterator_v =vnir_obj.iterate(by = 'line') iterator_s =swir_obj.iterate(by = 'line') while not iterator_v.complete: chunk_v = iterator_v.read_next()[:,3:] chunk_v =np.flip(chunk_v,axis=1) chunk_s = iterator_s.read_next()[:,:-6] chunk_s =np.flip(chunk_s,axis=1) if (iterator_v.current_line >=2) and (iterator_v.current_line <= 997): if (measurement == 'rdn') & shift: vnir_interpolator = interp1d(vnir_waves+shift_surface[iterator_v.current_line-2,:63], chunk_v[2:-2,:],fill_value = "extrapolate",kind=interp_kind) chunk_v = vnir_interpolator(vnir_waves) swir_interpolator = interp1d(swir_waves+shift_surface[iterator_v.current_line-2,63:], chunk_s[2:-2,:],fill_value = "extrapolate",kind=interp_kind) chunk_s = swir_interpolator(swir_waves) line = np.concatenate([chunk_v,chunk_s],axis=1)/1000. else: line = np.concatenate([chunk_v,chunk_s],axis=1)[2:-2,:]/1000. #Apply rad coeffs line*=coeff_arr[iterator_v.current_line-2,:] writer.write_line(line, iterator_v.current_line-2) #Load ancillary datasets geo = l1_obj['HDFEOS']["SWATHS"]['PRS_L1_HCO']['Geolocation Fields'] pvs = l1_obj['Info']["Ancillary"]['PVSdata'] # Time '''1. Convert from MJD2000 to UTC hours 2. Fit line to estimate continous time. ''' def dhour(day): epoch = dt.datetime(2000,1, 1,) epoch = epoch.replace(tzinfo=dt.timezone.utc) hour = (day-day//1)*24 minute = (hour-hour//1)*60 second= (minute-minute//1)*60 microsecond= (second-second//1)*1000000 time = epoch + dt.timedelta(days=day//1,hours=hour//1, minutes=minute//1,seconds=second, microseconds =microsecond) return time.hour + time.minute/60. + time.second/3600. v_dhour = np.vectorize(dhour) utc_time = v_dhour(np.array(geo['Time'][:])) utc_time = np.ones(geo['Longitude_VNIR'][:,:].shape[0]) *utc_time[:,np.newaxis] utc_time = utc_time[2:-2,2:-2] # Solar geometries '''Solar geometry is calculated based on the mean scene acquisition time which varies by less than 5 seconds from start to end of the scene and is computationally more efficient. ''' mjd2000_epoch = dt.datetime(2000,1, 1,) mjd2000_epoch = mjd2000_epoch.replace(tzinfo=dt.timezone.utc) mean_time = mjd2000_epoch + dt.timedelta(days=np.array(geo['Time'][:]).mean()) solar_az = solar.get_azimuth(geo['Latitude_VNIR'][:,:],geo['Longitude_VNIR'][:,:],mean_time)[2:-2,2:-2] solar_zn = 90-solar.get_altitude(geo['Latitude_VNIR'][:,:],geo['Longitude_VNIR'][:,:],mean_time)[2:-2,2:-2] longitude= geo['Longitude_VNIR'][2:-2,2:-2] latitude= geo['Latitude_VNIR'][2:-2,2:-2] #Create initial elevation raster elevation= dem_generate(longitude,latitude,elev_dir,temp_dir) zone,direction = utm_zone(longitude,latitude) # Calculate satellite X,Y,Z position for each line ''' GPS data are sampled at 1Hz resulting in steps in the position data, a line is fit to each dimension to estimate continuous position. There are more GPS samples than there are lines, to allign the GPS signal with the line, we use the provided 'Time' information for each line to match with the GPS data. When converting GPS time to UTC we use 17 sec difference instead of 18 sec because it matches the time provided in the time array. ''' # Convert satellite GPS position time to UTC sat_t = [] for second,week in zip(pvs['GPS_Time_of_Last_Position'][:].flatten(),pvs['Week_Number'][:].flatten()): gps_second = week*7*24*60*60 + second gps_epoch = dt.datetime(1980, 1, 6) gps_time = gps_epoch+ dt.timedelta(seconds=gps_second - 17) sat_t.append(gps_time.hour*3600 + gps_time.minute*60. + gps_time.second) sat_t = np.array(sat_t)[:,np.newaxis] # Convert line MJD2000 to UTC grd_t = [] for day in geo['Time'][:].flatten(): time = mjd2000_epoch + dt.timedelta(days=day) grd_t.append(time.hour*3600 + time.minute*60. + time.second) grd_t = np.array(grd_t)[:,np.newaxis] #Fit a line to ground time X = np.concatenate([np.arange(1000)[:,np.newaxis], np.ones(grd_t.shape)],axis=1) slope, intercept = np.linalg.lstsq(X,grd_t,rcond=-1)[0].flatten() line_t_linear = slope*np.arange(1000)+ intercept #Fit a line to satellite time measurements = np.arange(len(sat_t)) X = np.concatenate([measurements[:,np.newaxis], np.ones(sat_t.shape)],axis=1) slope, intercept = np.linalg.lstsq(X,sat_t,rcond=-1)[0].flatten() sat_t_linear = slope*measurements+ intercept # Interpolate x,y,z satelite positions sat_xyz = [] for sat_pos in ['x','y','z']: sat_p = np.array(pvs['Wgs84_pos_%s' % sat_pos][:]) slope, intercept = np.linalg.lstsq(X,sat_p,rcond=-1)[0].flatten() sat_p_linear = slope*measurements+ intercept interpolator = interp1d(sat_t_linear,sat_p_linear, fill_value="extrapolate",kind = 'linear') sat_interp = interpolator(line_t_linear) sat_xyz.append(sat_interp[2:-2]) sat_xyz = np.array(sat_xyz) # Calculate sensor to ground pathlength grd_xyz = np.array(dda2ecef(longitude,latitude,elevation)) path = pathlength(sat_xyz,grd_xyz) # Export satellite position to csv sat_lon,sat_lat,sat_alt = ecef2dda(sat_xyz[0],sat_xyz[1],sat_xyz[2]) satellite_df = pd.DataFrame() satellite_df['lat'] = sat_lat satellite_df['lon'] = sat_lon satellite_df['alt'] = sat_alt satellite_df.to_csv('%sPRS_%s_satellite_loc.csv' % (out_dir,base_name)) # Convert satellite coords to local ENU sat_enu = np.array(dda2utm(sat_lon,sat_lat,sat_alt, utm_zone(longitude,latitude))) # Convert ground coords to local ENU easting,northing,up =dda2utm(longitude,latitude, elevation) # Calculate sensor geometry sensor_zn,sensor_az = sensor_view_angles(sat_enu, np.array([easting,northing,up])) # Perform image matching if match: coords =np.concatenate([np.expand_dims(easting.flatten(),axis=1), np.expand_dims(northing.flatten(),axis=1)],axis=1) warp_east = easting.min()-100 warp_north =northing.max()+100 pixel_size = 30 project = Projector() project.create_tree(coords,easting.shape) project.query_tree(warp_east,warp_north,pixel_size) # Project independent variables sensor_az_prj = project.project_band(sensor_az,-9999,angular=True) sensor_zn_prj = project.project_band(sensor_zn,-9999,angular=True) elevation_prj = project.project_band(elevation.astype(np.float),-9999) radiance = ht.HyTools() radiance.read_file(rdn_file, 'envi') #Average over Landsat 8 Band 5 bandwidth and warp unwarp_band = np.zeros(longitude.shape) for wave in range(850,890,10): unwarp_band += radiance.get_wave(wave)/7. warp_band = project.project_band(unwarp_band,-9999) warp_band = 16000*(warp_band-warp_band.min())/warp_band.max() if isinstance(match,bool): landsat,land_east,land_north = get_landsat_image(longitude,latitude, mean_time.month, max_cloud = 5) else: lst = ht.HyTools() lst.read_file(match,'envi') landsat = lst.get_band(0) land_east = float(lst.map_info[3]) land_north = float(lst.map_info[4]) #Calculate offsets between reference and input images offset_x = int((warp_east-land_east)//pixel_size) offset_y = int((land_north-warp_north)//pixel_size) #Calculate optimal shift y_model,x_model = image_match(landsat,warp_band, offset_x,offset_y, sensor_zn_prj,sensor_az_prj,elevation_prj) #Apply uniform filter smooth_elevation = uniform_filter(elevation,25) smooth_az = uniform_filter(sensor_az,25) smooth_zn = uniform_filter(sensor_zn,25) # Generate y and x offset surfaces i,a,b,c = y_model y_offset = i + a*smooth_zn +b*smooth_az + c*smooth_elevation i,a,b,c= x_model x_offset = i + a*smooth_zn +b*smooth_az + c*smooth_elevation # Calculate updated coordinates easting = easting+ 30*x_offset northing = northing- 30*y_offset zone,direction = utm_zone(longitude,latitude) longitude,latitude = utm2dd(easting,northing,zone,direction) #Recalculate elevation with new coordinates logging.info('Rebuilding DEM') elevation= dem_generate(longitude,latitude,elev_dir,temp_dir) # Export location datacube loc_export(loc_file,longitude,latitude,elevation) # Generate remaining observable layers slope,aspect = slope_aspect(elevation,temp_dir) cosine_i = calc_cosine_i(np.radians(solar_zn), np.radians(solar_az), np.radians(slope), np.radians(aspect)) rel_az = np.radians(solar_az-sensor_az) phase = np.arccos(np.cos(np.radians(solar_zn)))*np.cos(np.radians(solar_zn)) phase += np.sin(np.radians(solar_zn))*np.sin(np.radians(solar_zn))*np.cos(rel_az) # Export observables datacube obs_export(obs_file,path,sensor_az,sensor_zn, solar_az,solar_zn,phase,slope,aspect, cosine_i,utc_time) if proj: #Create new projector with corrected coordinates new_coords =np.concatenate([np.expand_dims(easting.flatten(),axis=1), np.expand_dims(northing.flatten(),axis=1)],axis=1) project = Projector() project.create_tree(new_coords,easting.shape) project.query_tree(easting.min()-100,northing.max()+100,30) blocksize = int(res/30) map_info = ['UTM', 1, 1, easting.min()-100 - (res/2), northing.max()+100 + (res/2),res, res,zone,direction, 'WGS-84' , 'units=Meters'] out_cols = int(blocksize* (project.output_shape[1]//blocksize)) out_lines = int(blocksize* (project.output_shape[0]//blocksize)) logging.info('Georeferencing datasets to %sm resolution' % res) for file in ['rdn','loc','obs']: input_name = '%sPRS_%s_%s' % (temp_dir,base_name,file) hy_obj = ht.HyTools() hy_obj.read_file(input_name, 'envi') iterator =hy_obj.iterate(by = 'band') out_header = hy_obj.get_header() out_header['lines']= project.output_shape[0]//blocksize out_header['samples']=project.output_shape[1]//blocksize out_header['data ignore value'] = -9999 out_header['map info'] = map_info output_name = '%sPRS_%s_%s_prj' % (out_dir,base_name,file) writer = WriteENVI(output_name,out_header) while not iterator.complete: if (file == 'obs') & (iterator.current_band in [1,2,3,4,7]): angular = True else: angular = False band = project.project_band(iterator.read_next(),-9999,angular=angular) band[band == -9999] = np.nan bins =view_as_blocks(band[:out_lines,:out_cols], (blocksize,blocksize)) if angular: bins = np.radians(bins) band = circmean(bins,axis=2,nan_policy = 'omit') band = circmean(band,axis=2,nan_policy = 'omit') band = np.degrees(band) else: band = np.nanmean(bins,axis=(2,3)) if file == 'rdn': band[band<0] = 0 band[np.isnan(band)] = -9999 writer.write_band(band,iterator.current_band) logging.info('Deleting temporary files') shutil.rmtree(temp_dir)
iso_base = '%s/isotest/isofit/' % home figure_dir = '%s/sister/figures/prisma/radiance_factors/' % home doy = dt.datetime.strptime(base_name[:8],'%Y%m%d').timetuple().tm_yday base_file = '%s/data/prisma/rad/PRISMA_%s_rad_factor/PRISMA_%s_' % (home, base_name,base_name) cal_dir = '%s/temp/%s' % (home,base_name) if os.path.isdir(cal_dir): shutil.rmtree(cal_dir) os.mkdir(cal_dir) for subdir in ['/configs','/data','/output','/lut_full/']: os.mkdir(cal_dir+subdir) #Load datasets radiance = ht.HyTools() radiance.read_file(base_file + 'rad_unprj', 'envi') radiance.create_bad_bands([[300,380.0],[1300.0,1500],[1780.0,1975.0],[2450.0,2600]]) observables = ht.HyTools() observables.read_file(base_file + 'obs_unprj', 'envi') location = ht.HyTools() location.read_file(base_file + 'loc_unprj', 'envi') #Get center of image obs_mean = observables.get_chunk(0,-1,0,-1).mean(axis =(0,1)) loc_mean = location.get_chunk(0,-1,0,-1).mean(axis =(0,1)) # Export wavelength file wavelength_file = '%s/data/wavelengths.txt' % cal_dir
def resample(in_file, out_dir, resolution, verbose=True, unrotate=False): ''' Perform a two-step spatial resampling to . First, pixels are aggregated and averaged, next a nearest neighbor algorithm is used to resample images to resolution. ''' out_image = out_dir + '/' + os.path.basename(in_file) image = ht.HyTools() image.read_file(in_file, 'envi') x_ul = float(image.map_info[3]) y_ul = float(image.map_info[4]) pixel_res = float(image.map_info[5]) if 'rotation' in image.map_info[-1]: rotation = 360 + float(image.map_info[-1].split('=')[-1]) else: rotation = 0 # Calculate rotated and unrotated coords y_ind, x_ind = np.indices((image.lines, image.columns)) y_rcoord = y_ul - y_ind * pixel_res x_rcoord = x_ul + x_ind * pixel_res if unrotate: x_coord, y_coord = rotate_coords(x_rcoord, y_rcoord, x_ul, y_ul, rotation) else: x_coord, y_coord = x_rcoord, y_rcoord bin_size = int(np.round(resolution / pixel_res)) if verbose: print("Aggregating every %s pixels" % bin_size) lines = bin_size * (image.lines // bin_size) columns = bin_size * (image.columns // bin_size) y_coord_bin = np.nanmean(view_as_blocks(y_coord[:lines, :columns], (bin_size, bin_size)), axis=(2, 3)) x_coord_bin = np.nanmean(view_as_blocks(x_coord[:lines, :columns], (bin_size, bin_size)), axis=(2, 3)) # Get extent of output array xmax = int(resolution * (x_coord_bin.max() // resolution)) + resolution ymax = int(resolution * (y_coord_bin.max() // resolution)) + resolution xmin = int(resolution * (x_coord_bin.min() // resolution)) - resolution ymin = int(resolution * (y_coord_bin.min() // resolution)) - resolution out_columns = int((xmax - xmin) / resolution) out_lines = int((ymax - ymin) / resolution) #Calculate coordinates of output array image_shape = (out_lines, out_columns) y_coord_out, x_coord_out = np.indices(image_shape) * resolution y_coord_out = ymax - y_coord_out x_coord_out = xmin + x_coord_out #Create tree to convert pixels to geolocated pixels src_points = np.concatenate([ np.expand_dims(x_coord_bin.flatten(), axis=1), np.expand_dims(y_coord_bin.flatten(), axis=1) ], axis=1) tree = cKDTree(src_points, balanced_tree=True) dst_points = np.concatenate([ np.expand_dims(x_coord_out.flatten(), axis=1), np.expand_dims(y_coord_out.flatten(), axis=1) ], axis=1) dists, indexes = tree.query(dst_points, k=1) indices_int = np.unravel_index(indexes, x_coord_bin.shape) mask = dists.reshape(image_shape) > resolution out_header = image.get_header() out_header['lines'] = out_lines out_header['samples'] = out_columns out_header['map info'][3] = str(xmin) out_header['map info'][4] = str(ymax) out_header['map info'][5:7] = resolution, resolution if unrotate: out_header['map info'][-1] = 'rotation=0.0000' out_header['byte order'] = 0 out_header['data ignore value'] = image.no_data writer = WriteENVI(out_image, out_header) iterator = image.iterate(by='band') while not iterator.complete: if verbose & (iterator.current_band % 10 == 0): print("%s/%s" % (iterator.current_band, image.bands)) band = np.copy(iterator.read_next()).astype(float) band[~image.mask['no_data']] = np.nan bins = view_as_blocks(band[:lines, :columns], (bin_size, bin_size)) if (iterator.current_band in [1, 2, 3, 4, 7]) and ('obs' in image.base_name): bins = np.radians(bins) band = circmean(bins, axis=2, nan_policy='omit') band = circmean(band, axis=2, nan_policy='omit') band = np.degrees(band) else: band = np.nanmean(bins, axis=(2, 3)) band = band[indices_int[0], indices_int[1]].reshape(image_shape) band[mask] = image.no_data band[np.isnan(band)] = image.no_data writer.write_band(band, iterator.current_band)
def dem_generate(longitude, latitude, elev_dir, temp_dir): ''' Args: longitude (float): Longitude array latitude (float): Latitude array elev_dir (str): Directory of zipped elevation tiles temp_dir (str): Temporary output directory Returns: dem (np.array): Elevation array. ''' # Get extents of image lon_min = longitude.min() lon_max = longitude.max() lat_min = latitude.min() lat_max = latitude.max() if 'aws' in elev_dir: tiles = pd.read_csv(elev_dir + 'tileList.txt', header=None).values.flatten() else: tiles = glob.glob(elev_dir + '*.tar.gz') idx = index.Index(properties=index.Property()) #Get list of intersecting tiles for i, tile in enumerate(tiles): lat, ign, lon = os.path.basename(tile).replace('_COG', '').split('_')[3:6] if 'W' in lon: lon = -1 * float(lon[1:]) else: lon = float(lon[1:]) if 'S' in lat: lat = -1 * float(lat[1:]) else: lat = float(lat[1:]) idx.insert(i, (lon, lat, lon + 1, lat + 1)) tiles_intersect = [ tiles[n] for n in idx.intersection((lon_min, lat_min, lon_max, lat_max)) ] if len(tiles_intersect) == 0: constant_elev = float( input( "No overlapping tiles found, enter constant elevation for scene (m): " )) elevation = np.ones(longitude.shape) * constant_elev else: tile_string = "Found %s intersecting elevation tiles:" % len( tiles_intersect) for tile in tiles_intersect: tile_string += '\n\t%s' % tile if 'aws' in elev_dir: tile_url = "%s%s/%s.tif" % (elev_dir, tile, tile) tile_file = "%s%s.tif" % (temp_dir, tile) download_file(tile_file, tile_url) else: with tarfile.open(tile, 'r') as tar_ref: tar_ref.extractall(temp_dir) logging.info(tile_string) logging.info('Merging DEM tiles') dem_file = '%stemp_dem' % temp_dir os.system('gdal_merge.py -o %s -of ENVI %sCopernicus_DSM*' % (dem_file, temp_dir)) dem_obj = ht.HyTools() dem_obj.read_file(dem_file, 'envi') ulx = float(dem_obj.map_info[3]) uly = float(dem_obj.map_info[4]) pix = float(dem_obj.map_info[5]) dem_lat, dem_lon = np.indices((dem_obj.lines, dem_obj.columns)) dem_xl = int((lon_min - ulx) // pix) dem_xr = int((lon_max - ulx) // pix) dem_yu = int((uly - lat_max) // pix) dem_yd = int((uly - lat_min) // pix) dem_subset = dem_obj.get_chunk(dem_xl, dem_xr, dem_yu, dem_yd) dem_lat, dem_lon = np.indices(dem_subset.shape[:2]) dem_lat = (lat_max - dem_lat * pix).flatten() dem_lon = (lon_min + dem_lon * pix).flatten() #Create spatial index and nearest neighbor sample src_points = np.concatenate( [np.expand_dims(dem_lon, axis=1), np.expand_dims(dem_lat, axis=1)], axis=1) tree = cKDTree(src_points, balanced_tree=False) dst_points = np.concatenate([ longitude.flatten()[:, np.newaxis], latitude.flatten()[:, np.newaxis] ], axis=1) indexes = tree.query(dst_points, k=1)[1] indices_int = np.unravel_index(indexes, (dem_subset.shape[0], dem_subset.shape[1])) elevation = dem_subset[indices_int[0], indices_int[1]].reshape(longitude.shape) #Set negative elevations to 0 if np.sum(elevation < 0) > 0: logging.warning('Elevations below sea level found, setting to 0m') elevation[elevation < 0] = 0 return elevation
def l1b_process(l1b_zip, out_dir, temp_dir, elev_dir, match=None, proj=True, res=30): ''' This function exports three files: *_rad* : Merged and optionally shift corrected radiance cube *_obs* : Observables file in the format of JPL obs files: 1. Pathlength (m) 2. To-sensor view azimuth angle (degrees) 3. To-sensor view zenith angle (degrees) 4. To-sun azimuth angle (degrees) 5. To-sun zenith angle (degrees) 6. Phase 7. Slope (Degrees) 8. Aspect (Degrees) 9. Cosine i 10. UTC decimal hours *_loc* : Location file in the following format: 1. Longitude (decimal degrees) 2. Longitude (decimal degrees) 3. Elevation (m) l1(str): L1B zipped radiance data product path out_dir(str): Output directory of ENVI datasets temp_dir(str): Temporary directory for intermediate elev_dir (str): Directory zipped Copernicus elevation tiles or url to AWS Copernicus data ex : 'https://copernicus-dem-30m.s3.amazonaws.com/' match (str or list) : Pathname to Landsat image for image re-registration (recommended) proj (bool) : Project image to UTM grid res (int) : Resolution of projected image, 30 should be one of its factors (90,120,150.....) ''' base_name = os.path.basename(l1b_zip)[14:-4] out_dir = '%s/DESIS_%s/' % (out_dir, base_name) if not os.path.isdir(out_dir): os.mkdir(out_dir) temp_dir = '%s/tmpDESIS_%s/' % (temp_dir, base_name) if not os.path.isdir(temp_dir): os.mkdir(temp_dir) zip_base = os.path.basename(l1b_zip) logging.info('Unzipping %s' % zip_base) with zipfile.ZipFile(l1b_zip, 'r') as zipped: zipped.extractall(temp_dir) l1b_file = gdal.Open('%s/DESIS-HSI-L1B-%s-SPECTRAL_IMAGE.tif' % (temp_dir, base_name)) # Parse relevant metadata from XML file, assume metadata are in same directory as iamges tree = ET.parse('%s/DESIS-HSI-L1B-%s-METADATA.xml' % (temp_dir, base_name)) root = tree.getroot() specific = root[3] band_meta = {} for item in specific.findall('bandCharacterisation'): for band in item: for meta in band: if meta.tag not in band_meta.keys(): band_meta[meta.tag] = [] string = str(meta.text.encode('utf8')) string = string.replace('\\n', ' ') string = string.replace('b', ' ') string = string.replace("'", ' ') values = string.split(',') values = [float(x) for x in values] if len(values) == 1: values = values[0] band_meta[meta.tag].append(values) offset = np.array(band_meta['offsetOfBand']) gain = np.array(band_meta['gainOfBand']) waves = np.array(band_meta['wavelengthCenterOfBand']) fwhm = np.array(band_meta['wavelengthWidthOfBand']) response = np.array(band_meta['response']) response_waves = np.array(band_meta['wavelengths']) # Fit a gaussian to the reponse to determine center wavelength and fhwm opt_waves = [] opt_fwhm = [] for i, wave in enumerate(waves): popt, pcov = curve_fit(gaussian, response_waves[i], np.array(response[i]) / max(response[i]), [waves[i], fwhm[i]]) opt_waves.append(popt[0]) opt_fwhm.append(popt[1]) scene_az = float(specific.findall('sceneAzimuthAngle')[0].text) scene_zn = float(specific.findall('sceneIncidenceAngle')[0].text) # Get acquisition start and end time base = root[2] time_str = base.findall('temporalCoverage')[0].findall('startTime')[0].text time_str = time_str.replace('T', ' ').replace('Z', '') start_time = dt.datetime.strptime(time_str, "%Y-%m-%d %H:%M:%S.%f") start_time = start_time.replace(tzinfo=dt.timezone.utc) time_str = base.findall('temporalCoverage')[0].findall('endTime')[0].text time_str = time_str.replace('T', ' ').replace('Z', '') end_time = dt.datetime.strptime(time_str, "%Y-%m-%d %H:%M:%S.%f") end_time = end_time.replace(tzinfo=dt.timezone.utc) date = dt.datetime.strftime(start_time, "%Y%m%d") # Get orbital data orbit_data = [] for item in specific.findall('orbit'): for line in item: for point in line.findall('point'): time_str = line.findall('timeUTC')[0].text time_str = time_str.replace('T', ' ').replace('Z', '') orbit_time = dt.datetime.strptime(time_str, "%Y-%m-%d %H:%M:%S.%f") orbit_time = orbit_time.replace(tzinfo=dt.timezone.utc) #The metadata contains orbital info beyond the collection window # we use the acquisition start and end times to filter points if (orbit_time >= start_time) & (orbit_time <= end_time): for location in point.findall('location'): x = float(location.findall('X')[0].text) y = float(location.findall('Y')[0].text) z = float(location.findall('Z')[0].text) orbit_data.append([x, y, z]) orbit_data = np.array(orbit_data) l1b_band = l1b_file.ReadAsArray().mean(axis=0) # Get bounding coordinates of scene coord_dict = {} polygon = base.findall('spatialCoverage')[0].findall('boundingPolygon')[0] for point in polygon: name = point.findall('frame')[0].text lat = float(point.findall('latitude')[0].text) lon = float(point.findall('longitude')[0].text) coord_dict[name] = [lat, lon] # Get ISS altitude altitude_m = float(base.findall('altitudeCoverage')[0].text) raster = l1b_file.ReadAsArray() mask = raster[1].astype(float) mask = mask == mask[0][0] rad_dict = envi_header_dict() rad_dict['lines'] = l1b_file.RasterYSize rad_dict['samples'] = l1b_file.RasterXSize - 85 rad_dict['bands'] = len(waves) - 1 rad_dict['wavelength'] = opt_waves[1:] rad_dict['fwhm'] = opt_fwhm[1:] rad_dict['interleave'] = 'bil' rad_dict['data type'] = 4 rad_dict['wavelength units'] = "nanometers" rad_dict['byte order'] = 0 rad_dict['data ignore value'] = -9999 rad_dict['default bands'] = [ np.argmin(np.abs(waves - 660)), np.argmin(np.abs(waves - 560)), np.argmin(np.abs(waves - 460)) ] #Define output paths if proj: rad_file = '%sDESIS_%s_rdn' % (temp_dir, base_name) loc_file = '%sDESIS_%s_loc' % (temp_dir, base_name) obs_file = '%sDESIS_%s_obs' % (temp_dir, base_name) else: rad_file = '%sDESIS_%s_rdn' % (out_dir, base_name) loc_file = '%sDESIS_%s_loc' % (out_dir, base_name) obs_file = '%sDESIS_%s_obs' % (out_dir, base_name) writer = WriteENVI(rad_file, rad_dict) #Write VNIR cube logging.info('Exporting radiance data') for line_num in range(l1b_file.RasterYSize): line = raster[:, line_num, :].astype(float) line = line * gain[:, np.newaxis] + offset[:, np.newaxis] line = line[1:, 85:].T writer.write_line(line, line_num) del raster # Location datacube ########################################################################### lines, columns = np.indices((l1b_file.RasterYSize, l1b_file.RasterXSize)) lat_vals = [] lon_vals = [] points = [[0, 0], [l1b_file.RasterYSize, 0], [l1b_file.RasterYSize, l1b_file.RasterXSize], [0, l1b_file.RasterXSize]] for point in [1, 2, 3, 4]: lat_vals.append(coord_dict['point_%s' % point][0]) lon_vals.append(coord_dict['point_%s' % point][1]) longitude = griddata(points, lon_vals, (lines, columns), method='linear')[:, 85:] latitude = griddata(points, lat_vals, (lines, columns), method='linear')[:, 85:] #Create initial elevation raster elevation = dem_generate(longitude, latitude, elev_dir, temp_dir) zone, direction = utm_zone(longitude, latitude) solar_az = solar.get_azimuth(latitude, longitude, start_time) solar_zn = 90 - solar.get_altitude(latitude, longitude, start_time) ecef = pyproj.Proj(proj='geocent', ellps='WGS84', datum='WGS84') lla = pyproj.Proj(proj='latlong', ellps='WGS84', datum='WGS84') grd_xyz = np.array( pyproj.transform(lla, ecef, longitude, latitude, elevation, radians=False)) # Calculate satellite XYZ position sat_xyz = [] line_grid = np.linspace(0, 1, orbit_data.shape[0]) * longitude.shape[0] for sat_coord in orbit_data.T: interpolator = interp1d(line_grid, sat_coord) sat_interp = interpolator(np.arange(longitude.shape[0])) sat_xyz.append(sat_interp) sat_xyz = np.array(sat_xyz) path = np.linalg.norm(sat_xyz[:, :, np.newaxis] - grd_xyz, axis=0) # Export satellite position to csv sat_lon, sat_lat, sat_alt = ecef2dda(sat_xyz[0], sat_xyz[1], sat_xyz[2]) satellite_df = pd.DataFrame() satellite_df['lat'] = sat_lat satellite_df['lon'] = sat_lon satellite_df['alt'] = sat_alt satellite_df.to_csv('%sDESIS_%s_satellite_loc.csv' % (out_dir, base_name)) # Convert satellite coords to local ENU sat_enu = np.array( dda2utm(sat_lon, sat_lat, sat_alt, utm_zone(longitude, latitude))) # Convert ground coords to local ENU easting, northing, up = dda2utm(longitude, latitude, elevation) # Calculate sensor geometry sensor_zn, sensor_az = sensor_view_angles( sat_enu, np.array([easting, northing, up])) if match: coords = np.concatenate([ np.expand_dims(easting.flatten(), axis=1), np.expand_dims(northing.flatten(), axis=1) ], axis=1) warp_east = easting.min() - 100 warp_north = northing.max() + 100 pixel_size = 30 project = Projector() project.create_tree(coords, easting.shape) project.query_tree(warp_east, warp_north, pixel_size) # Project independent variables sensor_az_prj = project.project_band(sensor_az, -9999) sensor_zn_prj = project.project_band(sensor_zn, -9999) elevation_prj = project.project_band(elevation.astype(np.float), -9999) radiance = ht.HyTools() radiance.read_file(rad_file, 'envi') #Average over Landsat 8 Band 5 bandwidth and warp warp_band = np.zeros(longitude.shape) for wave in range(850, 890, 10): warp_band += radiance.get_wave(wave) / 7. warp_band = project.project_band(warp_band, -9999) warp_band = 16000 * (warp_band - warp_band.min()) / warp_band.max() landsat, land_east, land_north = get_landsat_image(longitude, latitude, end_time.month, max_cloud=5) #Calculate offsets between reference and input images offset_x = int((warp_east - land_east) // pixel_size) offset_y = int((land_north - warp_north) // pixel_size) #Calculate optimal shift y_model, x_model = image_match(landsat, warp_band, offset_x, offset_y, sensor_zn_prj, sensor_az_prj, elevation_prj, shift_max=30) #Apply uniform filter smooth_elevation = uniform_filter(elevation, 25) smooth_az = uniform_filter(sensor_az, 25) smooth_zn = uniform_filter(sensor_zn, 25) # Generate y and x offset surfaces i, a, b, c = y_model y_offset = i + a * smooth_zn + b * smooth_az + c * smooth_elevation i, a, b, c = x_model x_offset = i + a * smooth_zn + b * smooth_az + c * smooth_elevation # Calculate updated coordinates easting = easting + 30 * x_offset northing = northing - 30 * y_offset zone, direction = utm_zone(longitude, latitude) longitude, latitude = utm2dd(easting, northing, zone, direction) #Recalculate elevation with new coordinates logging.info('Rebuilding DEM') elevation = dem_generate(longitude, latitude, elev_dir, temp_dir) loc_export(loc_file, longitude, latitude, elevation) # Generate remaining observable layers slope, aspect = slope_aspect(elevation, temp_dir) cosine_i = calc_cosine_i(np.radians(solar_zn), np.radians(solar_az), np.radians(slope), np.radians(aspect)) rel_az = np.radians(solar_az - sensor_az) phase = np.arccos(np.cos(np.radians(solar_zn))) * np.cos( np.radians(solar_zn)) phase += np.sin(np.radians(solar_zn)) * np.sin( np.radians(solar_zn)) * np.cos(rel_az) utc_time = (lines / (l1b_file.RasterYSize) * (end_time - start_time).seconds) / 60 / 60 utc_time += start_time.hour + start_time.minute / 60 utc_time = utc_time[:, 85:] obs_export(obs_file, path, sensor_az, sensor_zn, solar_az, solar_zn, phase, slope, aspect, cosine_i, utc_time) if proj: #Create new projector with corrected coordinates new_coords = np.concatenate([ np.expand_dims(easting.flatten(), axis=1), np.expand_dims(northing.flatten(), axis=1) ], axis=1) project = Projector() project.create_tree(new_coords, easting.shape) project.query_tree(easting.min() - 100, northing.max() + 100, 30) blocksize = int(res / 30) map_info = [ 'UTM', 1, 1, easting.min() - 100, northing.max() + 100, res, res, zone, direction, 'WGS-84', 'units=Meters' ] out_cols = int(blocksize * (project.output_shape[1] // blocksize)) out_lines = int(blocksize * (project.output_shape[0] // blocksize)) logging.info('Georeferencing datasets') for file in ['rdn', 'loc', 'obs']: logging.info(file) input_name = '%sDESIS_%s_%s' % (temp_dir, base_name, file) hy_obj = ht.HyTools() hy_obj.read_file(input_name, 'envi') iterator = hy_obj.iterate(by='band') out_header = hy_obj.get_header() out_header['lines'] = project.output_shape[0] // blocksize out_header['samples'] = project.output_shape[1] // blocksize out_header['data ignore value'] = -9999 out_header['map info'] = map_info output_name = '%sDESIS_%s_%s_prj' % (out_dir, base_name, file) writer = WriteENVI(output_name, out_header) while not iterator.complete: band = project.project_band(iterator.read_next(), -9999) band[band == -9999] = np.nan band = np.nanmean(view_as_blocks(band[:out_lines, :out_cols], (blocksize, blocksize)), axis=(2, 3)) if file == 'rdn': band[band < 0] = 0 band[np.isnan(band)] = -9999 writer.write_band(band, iterator.current_band) logging.info('Deleting temporary files') shutil.rmtree(temp_dir)
def h5_radiance_to_envi(filename, resolution=1): '''Convert a NEON HDF radiance file to ENVI formated image along with observables and location data cubes TODO: Recalculate terrain azimuth, provided product may be incorrect Args: filename (str): Path to HDF file. resolution (int, optional): Output image resolution. Defaults to 1. Returns: None. ''' # Load HDF file hdf_obj = h5py.File(filename, 'r') key = [key for key in hdf_obj.keys()][0] rad_dec = hdf_obj[key]['Radiance']['RadianceDecimalPart'] rad_int = hdf_obj[key]['Radiance']['RadianceIntegerPart'] obs = hdf_obj[key]['Radiance']['Metadata']['Ancillary_Rasters']['OBS_Data'] igm = hdf_obj[key]['Radiance']['Metadata']['Ancillary_Rasters']['IGM_Data'] wavelengths = hdf_obj[key]['Radiance']['Metadata']['Spectral_Data'][ 'Wavelength'][:].tolist() fwhm = hdf_obj[key]['Radiance']['Metadata']['Spectral_Data'][ 'FWHM'][:].tolist() map_info = hdf_obj[key]['Radiance']['Metadata']['Coordinate_System'][ 'Map_Info'][()].decode("utf-8").split(',') epsg = hdf_obj[key]['Radiance']['Metadata']['Coordinate_System'][ 'EPSG Code'][()].decode("utf-8") new_lines = rad_dec.shape[0] // resolution new_cols = rad_dec.shape[1] // resolution map_info[5] = resolution map_info[6] = resolution map_info = [str(info).strip() for info in map_info] # Export integer and decimal radiance components # to temporary ENVI files rad_dict = envi_header_dict() rad_dict['lines'] = rad_dec.shape[0] rad_dict['samples'] = rad_dec.shape[1] rad_dict['bands'] = rad_dec.shape[2] rad_dict['interleave'] = 'bsq' rad_dict['data type'] = 12 rad_dict['byte order'] = 0 dec_temp = filename.replace('radiance.h5', 'rad_dec') writer = WriteENVI(dec_temp, rad_dict) writer.write_chunk(rad_dec, 0, 0) int_temp = filename.replace('radiance.h5', 'rad_int') writer = WriteENVI(int_temp, rad_dict) writer.write_chunk(rad_int, 0, 0) int_obj = ht.HyTools() int_obj.read_file(int_temp, 'envi') dec_obj = ht.HyTools() dec_obj.read_file(dec_temp, 'envi') # Export radiance ################## rad_dict = envi_header_dict() rad_dict['lines'] = new_lines rad_dict['samples'] = new_cols rad_dict['bands'] = rad_dec.shape[2] rad_dict['wavelength'] = wavelengths rad_dict['fwhm'] = fwhm rad_dict['interleave'] = 'bil' rad_dict['data type'] = 4 rad_dict['wavelength units'] = "nanometers" rad_dict['byte order'] = 0 rad_dict['data ignore value'] = -9999 rad_dict['map info'] = map_info output_name = filename.replace('radiance.h5', 'rad') writer = WriteENVI(output_name, rad_dict) for band_num in range(rad_dict['bands']): print(band_num) band_int = int_obj.get_band(band_num).astype(float) band_dec = dec_obj.get_band(band_num) / 50000 band = band_int + band_dec band[band_int == 255] = np.nan band = band[:new_lines * resolution, :new_cols * resolution] band = view_as_blocks(band, (resolution, resolution)).mean(axis=(2, 3)) band[np.isnan(band)] = -9999 writer.write_band(band, band_num) os.remove(dec_temp) os.remove(int_temp) # Export observables #################### obs_dict = envi_header_dict() obs_dict['band_names'] = [ 'path length', 'to-sensor azimuth', 'to-sensor zenith', 'to-sun azimuth', 'to-sun zenith', 'phase', 'slope', 'aspect', 'cosine i', 'UTC time' ] obs_dict['data type'] = 4 obs_dict['lines'] = new_lines obs_dict['samples'] = new_cols obs_dict['bands'] = 10 obs_dict['fwhm'] = fwhm obs_dict['interleave'] = 'bil' obs_dict['data type'] = 4 obs_dict['byte order'] = 0 obs_dict['data ignore value'] = -9999 obs_dict['map info'] = map_info output_name = filename.replace('radiance.h5', 'obs') writer = WriteENVI(output_name, obs_dict) for band_num in range(obs_dict['bands']): print(band_num) band = obs[:, :, band_num] band[band == -9999] = np.nan band = band[:new_lines * resolution, :new_cols * resolution] band = view_as_blocks(band, (resolution, resolution)).mean(axis=(2, 3)) band[np.isnan(band)] = -9999 writer.write_band(band, band_num) # Export location datacube (lon,lat,elevation) ############################################## loc_dict = envi_header_dict() loc_dict['band_names'] = ['longitude', 'latitude', 'elevation'] loc_dict['data type'] = 4 loc_dict['lines'] = new_lines loc_dict['samples'] = new_cols loc_dict['bands'] = 3 loc_dict['fwhm'] = fwhm loc_dict['interleave'] = 'bil' loc_dict['data type'] = 4 loc_dict['byte order'] = 0 loc_dict['data ignore value'] = -9999 loc_dict['map info'] = map_info output_name = filename.replace('radiance.h5', 'loc') writer = WriteENVI(output_name, loc_dict) in_proj = pyproj.Proj("+init=EPSG:%s" % epsg) out_proj = pyproj.Proj("+init=EPSG:4326") longitude, latitude = pyproj.transform(in_proj, out_proj, igm[:, :, 0], igm[:, :, 1]) elevation = igm[:, :, 2] mask = elevation == -9999 for band_num, band in enumerate([longitude, latitude, elevation]): print(band_num) band[mask] = np.nan band = band[:new_lines * resolution, :new_cols * resolution] band = view_as_blocks(band, (resolution, resolution)).mean(axis=(2, 3)) band[np.isnan(band)] = -9999 writer.write_band(band, band_num) os.remove(filename)
def reflectance(self): '''Run ISOFIT reflectance inversion ''' # Check for input files if not (self.exists(self.rdn_file) & self.exists(self.loc_file) & self.exists(self.obs_file)): print('Radiance data not found, generating radiance datasets') self.radiance() rfl_tmp = self.tmp_dir + '%s_isofit/' % self.base_name if not os.path.isdir(rfl_tmp): os.mkdir(rfl_tmp) else: shutil.rmtree(rfl_tmp) os.mkdir(rfl_tmp) for key in self.isofit['surface']: get_surface_spectra(rfl_tmp + key, self.isofit['surface'][key]) surface_config = "%s/surface_config.json" % rfl_tmp surface_file = '%s/surface_filtered.mat'% rfl_tmp wavelength_file = gen_wavelength_file(self.rdn_file, rfl_tmp) surface_config_gen(rfl_tmp, self.isofit['surface_type'], self.isofit['windows'], wavelength_file, surface_file, surface_config) surface_model(surface_config) apply_oe = ['python'] apply_oe.append('%s/isofit/utils/apply_oe.py' % self.isofit['base']) apply_oe.append(self.rdn_file) apply_oe.append(self.loc_file) apply_oe.append(self.obs_file) apply_oe.append(rfl_tmp) if self.sensor in ['prisma','desis']: apply_oe.append('NA-%s' % self.date) apply_oe += ['--surface_path', surface_file] apply_oe += ['--n_cores',str(cpu_count())] apply_oe += ['--empirical_line',str(self.isofit['empirical_line'])] apply_oe += ['--presolve',str(self.isofit['presolve'])] apply_oe += ['--ray_temp_dir',rfl_tmp] apply_oe += ['--log_file','%s/%s_logfile' % (rfl_tmp,self.base_name)] apply_oe += ['--emulator_base',self.isofit['emulator']] if self.rdn_cfg['radiance_factors']: rad_factors = "%s/radiance_factors.txt" % rfl_tmp download_file(rad_factors,self.rdn_cfg['radiance_factors']) apply_oe += ['--rdn_factors_path',rad_factors] apply = subprocess.Popen(apply_oe) apply.wait() if not os.path.isdir(os.path.dirname(self.rfl_file)): os.mkdir(os.path.dirname(self.rfl_file)) # Mask windows and rename ISOFIT output files rfl = glob.glob('%s/output/*_rfl_prj_rfl' % rfl_tmp)[0] uncert = glob.glob('%s/output/*_uncert_prj_uncert'% rfl_tmp)[0] for new,old in [(self.rfl_file,rfl),(self.unc_file,uncert)]: hy_obj = ht.HyTools() hy_obj.read_file(old,'envi') mask = [] for wave in hy_obj.wavelengths: window = False for start,end in self.isofit['windows']: if (wave>start) and (wave<end): window |= True mask.append(window) header_dict = hy_obj.get_header() writer = ht.io.WriteENVI(new, header_dict) for band_num in range(hy_obj.bands): band = hy_obj.get_band(band_num) if not mask[band_num]: band = np.zeros(band.shape) writer.write_band(band,band_num) shutil.rmtree(rfl_tmp)
def preprocess(input_tar,out_dir,temp_dir,res = 0): ''' input_tar = '/data2/avcl/raw/f080709t01p00r15.tar.gz' input_tar = '/data2/avng/rdn/ang20170901t195659.tar.gz' out_dir ='/data1/temp/ang_pre/output/' temp_dir ='/data1/temp/ang_pre/temp/' ''' base_name = os.path.basename(input_tar) if base_name.startswith('ang'): base_name = base_name[:18] elif base_name.startswith('f'): base_name = base_name[:16] else: raise ValueError('Unrecognized sensor') out_dir = "%s/%s/" % (out_dir,base_name) if not os.path.isdir(out_dir): os.mkdir(out_dir) file = tarfile.open(input_tar) tar_contents = [temp_dir+c for c in file.getnames()] file.extractall(temp_dir) file.close() #AVIRIS NG if base_name.startswith('ang'): obs_ort_file = [x for x in tar_contents if x.endswith('obs_ort')][0] rdn_file = [x for x in tar_contents if x.endswith('img')][0] loc_file = [x for x in tar_contents if x.endswith('loc')][0] glt_file = [x for x in tar_contents if x.endswith('glt')][0] create_loc_ort(loc_file,glt_file) time_correct(obs_ort_file) #AVIRIS Classic elif base_name.startswith('f'): obs_ort_file = [x for x in tar_contents if x.endswith('obs_ort')][0] rdn_file = [x for x in tar_contents if x.endswith('ort_img')][0] igm_file = [x for x in tar_contents if x.endswith('igm')][0] glt_file = [x for x in tar_contents if x.endswith('ort_glt')][0] #Rename files for consistency loc_file = igm_file.replace('igm','loc') os.rename(igm_file,loc_file) os.rename(igm_file+'.hdr',loc_file+'.hdr') #Scale radiance os.rename(rdn_file,rdn_file+'_unscale') os.rename(rdn_file + '.hdr',rdn_file+'_unscale.hdr') rdn = ht.HyTools() rdn.read_file(rdn_file+'_unscale','envi') prefix = rdn.base_name[:3] if prefix in ['f95', 'f96', 'f97', 'f98', 'f99', 'f00', 'f01', 'f02', 'f03', 'f04', 'f05']: gains = np.r_[50.0 * np.ones(160), 100.0 * np.ones(64)] elif prefix in ['f06', 'f07', 'f08', 'f09', 'f10', 'f11', 'f12', 'f13', 'f14', 'f15', 'f16', 'f17', 'f18', 'f19', 'f20', 'f21']: gains = np.r_[300.0 * np.ones(110), 600.0 * np.ones(50), 1200.0 * np.ones(64)] rdn_header = rdn.get_header() rdn_header['byte order'] = 0 rdn_header['no data value'] = -9999 rdn_header['data type'] = 4 writer = WriteENVI(rdn.file_name.replace('_unscale',''),rdn_header) iterator =rdn.iterate(by = 'band') while not iterator.complete: band = np.copy(iterator.read_next()).astype(float) band /= gains[iterator.current_band] band[~rdn.mask['no_data']] = -9999 writer.write_band(band,iterator.current_band) create_loc_ort(loc_file,glt_file) loc_ort_file = loc_file+'_ort' for file in [obs_ort_file,rdn_file,loc_ort_file]: if res==0: shutil.move(file,out_dir) shutil.move(file + '.hdr',out_dir) else: resample(file,out_dir,res) shutil.rmtree(tar_contents[0])