def slope_aspect(elevation, temp_dir):
''' Use GDAL to calculte slope and aspect
'''
dem_dict = envi_header_dict()
dem_dict['lines'] = elevation.shape[0]
dem_dict['samples'] = elevation.shape[1]
dem_dict['bands'] = 1
dem_dict['interleave'] = 'bsq'
dem_dict['data type'] = 4
dem_file = '%stemp_dem_clip' % temp_dir
writer = WriteENVI(dem_file, dem_dict)
writer.write_band(elevation, 0)
slope_file = '%s_slope' % temp_dir
aspect_file = '%s_aspect' % temp_dir
logging.info('Calculating slope')
os.system('gdaldem slope -of ENVI %s %s' % (dem_file, slope_file))
logging.info('Calculating aspect')
os.system('gdaldem aspect -f ENVI %s %s' % (dem_file, aspect_file))
asp_obj = ht.HyTools()
asp_obj.read_file(aspect_file, 'envi')
aspect = asp_obj.get_band(0)
slp_obj = ht.HyTools()
slp_obj.read_file(slope_file, 'envi')
slope = slp_obj.get_band(0)
return slope, aspect
def gen_wavelength_file(rad_file, out_dir):
'''Export radiance image wavelengths and FWHM to file
'''
radiance = ht.HyTools()
radiance.read_file(rad_file, 'envi')
wave_fit = pd.DataFrame(index=[x for x in range(radiance.bands)])
wave_fit[0] = np.round(radiance.wavelengths / 1000, 5)
wave_fit[1] = np.round(radiance.fwhm / 1000, 5)
wavelength_file = '%s/wavelength_fit.txt' % out_dir
wave_fit.to_csv(wavelength_file, sep='\t', header=False)
return wavelength_file
def create_loc_ort(loc_file,glt_file):
loc = ht.HyTools()
loc.read_file(loc_file,'envi')
glt = ht.HyTools()
glt.read_file(glt_file,'envi')
samples = np.abs(glt.get_band(0))
lines = np.abs(glt.get_band(1))
lon = np.copy(loc.get_band(0))
lat= np.copy(loc.get_band(1))
elv = np.copy(loc.get_band(2))
lon_proj = lon[lines.flatten()-1,samples.flatten()-1]
lon_proj = lon_proj.reshape(lines.shape)
lat_proj = lat[lines.flatten()-1,samples.flatten()-1]
lat_proj = lat_proj.reshape(lines.shape)
elv_proj = elv[lines.flatten()-1,samples.flatten()-1]
elv_proj = elv_proj.reshape(lines.shape)
lon_proj[samples ==0] = -9999
lat_proj[samples ==0] = -9999
elv_proj[samples ==0] = -9999
# Create output file
header_dict =glt.get_header()
header_dict['bands'] = 3
header_dict['data type'] = 4
header_dict['band names'] = loc.get_header()['band names']
header_dict['description'] = loc.get_header()['description']
writer = WriteENVI(loc_file +'_ort',header_dict)
writer.write_band(lon_proj,0)
writer.write_band(lat_proj,1)
writer.write_band(elv_proj,2)
writer.close()
def time_correct(obs_ort_file):
obs = ht.HyTools()
obs.read_file(obs_ort_file,'envi')
utc_time = obs.get_band(9)[obs.mask['no_data']]
if (utc_time.max() > 24) | (utc_time.min() < 0):
hour = np.ones((obs.lines,obs.columns))
hour += int(obs.base_name[12:14])
hour += int(obs.base_name[14:16])/60
hour += int(obs.base_name[16:18])/3600
hour[~obs.mask['no_data']] = obs.no_data
obs.load_data(mode = 'r+')
if obs.interleave == 'bip':
obs.data[:,:,9] =hour
elif obs.interleave == 'bil':
obs.data[:,9,:] =hour
elif obs.interleave == 'bsq':
obs.data[9,:,:] =hour
obs.close_data()
def he5_to_envi(l1_zip,out_dir,temp_dir,elev_dir,shift = False, rad_coeff = None,
match=False,proj = False,res = 30):
'''
This function exports three files:
*_rdn* : Merged and optionally shift corrected radiance cube
*_obs* : Observables file in the format of JPL obs files:
1. Pathlength (m)
2. To-sensor view azimuth angle (degrees)
3. To-sensor view zenith angle (degrees)
4. To-sun azimuth angle (degrees)
5. To-sun zenith angle (degrees)
6. Phase
7. Slope (Degrees)
8. Aspect (Degrees)
9. Cosine i
10. UTC decimal hours
*_loc* : Location file in the following format:
1. Longitude (decimal degrees)
2. Longitude (decimal degrees)
3. Elevation (m)
l1(str): L1 zipped radiance data product path
out_dir(str): Output directory of ENVI datasets
temp_dir(str): Temporary directory for intermediate
elev_dir (str): Directory zipped Copernicus elevation tiles or url to AWS Copernicus data
ex : 'https://copernicus-dem-30m.s3.amazonaws.com/'
shift (bool) : Apply wavelength shift correction surface file
rad_coeff (bool) : Apply radiometric correction coefficients file
match (bool or string) : Perform landsat image matching, if string path to reference file
proj (bool) : Project image to UTM grid
res (int) : Resolution of projected image, 30 should be one of its factors (90,120,150.....)
'''
base_name = os.path.basename(l1_zip)[16:-4]
out_dir = "%s/PRS_%s/" % (out_dir,base_name)
if not os.path.isdir(out_dir):
os.mkdir(out_dir)
logging.basicConfig(filename='%s/PRS_%s.log' % (out_dir,base_name),
format='%(asctime)s: %(levelname)s - %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
level=logging.NOTSET)
temp_dir = '%s/tmpPRS_%s/'% (temp_dir,base_name)
if not os.path.isdir(temp_dir):
os.mkdir(temp_dir)
zip_base =os.path.basename(l1_zip)
logging.info('Unzipping %s' % zip_base)
with zipfile.ZipFile(l1_zip,'r') as zipped:
zipped.extractall(temp_dir)
l1_obj = h5py.File('%sPRS_L1_STD_OFFL_%s.he5' % (temp_dir,base_name),'r')
if shift:
shift_file = importlib.resources.open_binary(data,"PRS_20210409105743_20210409105748_0001_wavelength_shift_surface.npz")
shift_obj = np.load(shift_file)
shift_surface = shift_obj['shifts']
#interp_kind = shift_obj['interp_kind']
interp_kind='quadratic'
coeff_arr = np.ones((996, 230))
if rad_coeff != None:
coeff_file = importlib.resources.open_binary(data,"PRS_20210409105743_20210409105748_0001_radcoeff_surface.npz")
coeff_obj = np.load(coeff_file)
if rad_coeff == 'full':
coeff_arr = coeff_obj['coeffs']
elif rad_coeff == 'mean':
coeff_arr[:] = coeff_obj['coeffs'].mean(axis=0)
elif rad_coeff == 'center':
coeff_arr[:] = coeff_obj['coeffs'][498-25:498+25].mean(axis=0)
else:
print('Unrecognized coeff type')
#Define output paths
if proj:
rdn_file = '%sPRS_%s_rdn' % (temp_dir,base_name)
loc_file = '%sPRS_%s_loc' % (temp_dir,base_name)
obs_file = '%sPRS_%s_obs' % (temp_dir,base_name)
else:
rdn_file = '%sPRS_%s_rdn' % (out_dir,base_name)
loc_file = '%sPRS_%s_loc' % (out_dir,base_name)
obs_file = '%sPRS_%s_obs' % (out_dir,base_name)
measurement = 'rdn'
logging.info('Exporting radiance data')
# Export VNIR to temporary ENVI
vnir_data = l1_obj['HDFEOS']["SWATHS"]['PRS_L1_HCO']['Data Fields']['VNIR_Cube']
vnir_waves = l1_obj.attrs.get('List_Cw_Vnir')
vnir_fwhm = l1_obj.attrs.get('List_Fwhm_Vnir')
rdn_dict = envi_header_dict ()
rdn_dict['lines']= vnir_data.shape[0]
rdn_dict['samples']= vnir_data.shape[2]
rdn_dict['bands']= vnir_data.shape[1]
rdn_dict['wavelength']= vnir_waves
rdn_dict['fwhm']= vnir_fwhm
rdn_dict['interleave']= 'bsq'
rdn_dict['data type'] = 12
rdn_dict['wavelength units'] = "nanometers"
rdn_dict['byte order'] = 0
vnir_temp = '%sPRS_%s_%s_vnir' % (temp_dir,base_name,measurement)
writer = WriteENVI(vnir_temp,rdn_dict )
writer.write_chunk(np.moveaxis(vnir_data[:,:,:],1,2), 0,0)
# Export SWIR to temporary ENVI
swir_data = l1_obj['HDFEOS']["SWATHS"]['PRS_L1_HCO']['Data Fields']['SWIR_Cube']
swir_waves = l1_obj.attrs.get('List_Cw_Swir')
swir_fwhm = l1_obj.attrs.get('List_Fwhm_Swir')
rdn_dict['lines']= swir_data.shape[0]
rdn_dict['samples']= swir_data.shape[2]
rdn_dict['bands']= swir_data.shape[1]
rdn_dict['wavelength']= swir_waves
rdn_dict['fwhm']= swir_fwhm
swir_temp = '%sPRS_%s_%s_swir' % (temp_dir,base_name,measurement)
writer = WriteENVI(swir_temp,rdn_dict )
writer.write_chunk(np.moveaxis(swir_data[:,:,:],1,2), 0,0)
vnir_waves = np.flip(vnir_waves[3:]) #6
swir_waves = np.flip(swir_waves[:-6]) #-3
vnir_fwhm = np.flip(vnir_fwhm[3:])
swir_fwhm = np.flip(swir_fwhm[:-6])
vnir_obj = ht.HyTools()
vnir_obj.read_file(vnir_temp, 'envi')
swir_obj = ht.HyTools()
swir_obj.read_file(swir_temp, 'envi')
rdn_dict = envi_header_dict()
rdn_dict ['lines']= vnir_obj.lines-4 #Clip edges of array
rdn_dict ['samples']=vnir_obj.columns-4 #Clip edges of array
rdn_dict ['bands']= len(vnir_waves.tolist() + swir_waves.tolist())
rdn_dict ['wavelength']= vnir_waves.tolist() + swir_waves.tolist()
rdn_dict ['fwhm']= vnir_fwhm.tolist() + swir_fwhm.tolist()
rdn_dict ['interleave']= 'bil'
rdn_dict ['data type'] = 4
rdn_dict ['wavelength units'] = "nanometers"
rdn_dict ['byte order'] = 0
rdn_dict ['default bands'] = [int(vnir_obj.wave_to_band(660)),
int(vnir_obj.wave_to_band(560)),
int(vnir_obj.wave_to_band(460))]
writer = WriteENVI(rdn_file,rdn_dict)
iterator_v =vnir_obj.iterate(by = 'line')
iterator_s =swir_obj.iterate(by = 'line')
while not iterator_v.complete:
chunk_v = iterator_v.read_next()[:,3:]
chunk_v =np.flip(chunk_v,axis=1)
chunk_s = iterator_s.read_next()[:,:-6]
chunk_s =np.flip(chunk_s,axis=1)
if (iterator_v.current_line >=2) and (iterator_v.current_line <= 997):
if (measurement == 'rdn') & shift:
vnir_interpolator = interp1d(vnir_waves+shift_surface[iterator_v.current_line-2,:63],
chunk_v[2:-2,:],fill_value = "extrapolate",kind=interp_kind)
chunk_v = vnir_interpolator(vnir_waves)
swir_interpolator = interp1d(swir_waves+shift_surface[iterator_v.current_line-2,63:],
chunk_s[2:-2,:],fill_value = "extrapolate",kind=interp_kind)
chunk_s = swir_interpolator(swir_waves)
line = np.concatenate([chunk_v,chunk_s],axis=1)/1000.
else:
line = np.concatenate([chunk_v,chunk_s],axis=1)[2:-2,:]/1000.
#Apply rad coeffs
line*=coeff_arr[iterator_v.current_line-2,:]
writer.write_line(line, iterator_v.current_line-2)
#Load ancillary datasets
geo = l1_obj['HDFEOS']["SWATHS"]['PRS_L1_HCO']['Geolocation Fields']
pvs = l1_obj['Info']["Ancillary"]['PVSdata']
# Time
'''1. Convert from MJD2000 to UTC hours
2. Fit line to estimate continous time.
'''
def dhour(day):
epoch = dt.datetime(2000,1, 1,)
epoch = epoch.replace(tzinfo=dt.timezone.utc)
hour = (day-day//1)*24
minute = (hour-hour//1)*60
second= (minute-minute//1)*60
microsecond= (second-second//1)*1000000
time = epoch + dt.timedelta(days=day//1,hours=hour//1,
minutes=minute//1,seconds=second,
microseconds =microsecond)
return time.hour + time.minute/60. + time.second/3600.
v_dhour = np.vectorize(dhour)
utc_time = v_dhour(np.array(geo['Time'][:]))
utc_time = np.ones(geo['Longitude_VNIR'][:,:].shape[0]) *utc_time[:,np.newaxis]
utc_time = utc_time[2:-2,2:-2]
# Solar geometries
'''Solar geometry is calculated based on the mean scene acquisition time
which varies by less than 5 seconds from start to end of the scene and is
computationally more efficient.
'''
mjd2000_epoch = dt.datetime(2000,1, 1,)
mjd2000_epoch = mjd2000_epoch.replace(tzinfo=dt.timezone.utc)
mean_time = mjd2000_epoch + dt.timedelta(days=np.array(geo['Time'][:]).mean())
solar_az = solar.get_azimuth(geo['Latitude_VNIR'][:,:],geo['Longitude_VNIR'][:,:],mean_time)[2:-2,2:-2]
solar_zn = 90-solar.get_altitude(geo['Latitude_VNIR'][:,:],geo['Longitude_VNIR'][:,:],mean_time)[2:-2,2:-2]
longitude= geo['Longitude_VNIR'][2:-2,2:-2]
latitude= geo['Latitude_VNIR'][2:-2,2:-2]
#Create initial elevation raster
elevation= dem_generate(longitude,latitude,elev_dir,temp_dir)
zone,direction = utm_zone(longitude,latitude)
# Calculate satellite X,Y,Z position for each line
''' GPS data are sampled at 1Hz resulting in steps in the
position data, a line is fit to each dimension to estimate
continuous position.
There are more GPS samples than there are lines, to allign
the GPS signal with the line, we use the provided 'Time'
information for each line to match with the GPS data.
When converting GPS time to UTC we use 17 sec difference
instead of 18 sec because it matches the time provided in
the time array.
'''
# Convert satellite GPS position time to UTC
sat_t = []
for second,week in zip(pvs['GPS_Time_of_Last_Position'][:].flatten(),pvs['Week_Number'][:].flatten()):
gps_second = week*7*24*60*60 + second
gps_epoch = dt.datetime(1980, 1, 6)
gps_time = gps_epoch+ dt.timedelta(seconds=gps_second - 17)
sat_t.append(gps_time.hour*3600 + gps_time.minute*60. + gps_time.second)
sat_t = np.array(sat_t)[:,np.newaxis]
# Convert line MJD2000 to UTC
grd_t = []
for day in geo['Time'][:].flatten():
time = mjd2000_epoch + dt.timedelta(days=day)
grd_t.append(time.hour*3600 + time.minute*60. + time.second)
grd_t = np.array(grd_t)[:,np.newaxis]
#Fit a line to ground time
X = np.concatenate([np.arange(1000)[:,np.newaxis], np.ones(grd_t.shape)],axis=1)
slope, intercept = np.linalg.lstsq(X,grd_t,rcond=-1)[0].flatten()
line_t_linear = slope*np.arange(1000)+ intercept
#Fit a line to satellite time
measurements = np.arange(len(sat_t))
X = np.concatenate([measurements[:,np.newaxis], np.ones(sat_t.shape)],axis=1)
slope, intercept = np.linalg.lstsq(X,sat_t,rcond=-1)[0].flatten()
sat_t_linear = slope*measurements+ intercept
# Interpolate x,y,z satelite positions
sat_xyz = []
for sat_pos in ['x','y','z']:
sat_p = np.array(pvs['Wgs84_pos_%s' % sat_pos][:])
slope, intercept = np.linalg.lstsq(X,sat_p,rcond=-1)[0].flatten()
sat_p_linear = slope*measurements+ intercept
interpolator = interp1d(sat_t_linear,sat_p_linear,
fill_value="extrapolate",kind = 'linear')
sat_interp = interpolator(line_t_linear)
sat_xyz.append(sat_interp[2:-2])
sat_xyz = np.array(sat_xyz)
# Calculate sensor to ground pathlength
grd_xyz = np.array(dda2ecef(longitude,latitude,elevation))
path = pathlength(sat_xyz,grd_xyz)
# Export satellite position to csv
sat_lon,sat_lat,sat_alt = ecef2dda(sat_xyz[0],sat_xyz[1],sat_xyz[2])
satellite_df = pd.DataFrame()
satellite_df['lat'] = sat_lat
satellite_df['lon'] = sat_lon
satellite_df['alt'] = sat_alt
satellite_df.to_csv('%sPRS_%s_satellite_loc.csv' % (out_dir,base_name))
# Convert satellite coords to local ENU
sat_enu = np.array(dda2utm(sat_lon,sat_lat,sat_alt,
utm_zone(longitude,latitude)))
# Convert ground coords to local ENU
easting,northing,up =dda2utm(longitude,latitude,
elevation)
# Calculate sensor geometry
sensor_zn,sensor_az = sensor_view_angles(sat_enu,
np.array([easting,northing,up]))
# Perform image matching
if match:
coords =np.concatenate([np.expand_dims(easting.flatten(),axis=1),
np.expand_dims(northing.flatten(),axis=1)],axis=1)
warp_east = easting.min()-100
warp_north =northing.max()+100
pixel_size = 30
project = Projector()
project.create_tree(coords,easting.shape)
project.query_tree(warp_east,warp_north,pixel_size)
# Project independent variables
sensor_az_prj = project.project_band(sensor_az,-9999,angular=True)
sensor_zn_prj = project.project_band(sensor_zn,-9999,angular=True)
elevation_prj = project.project_band(elevation.astype(np.float),-9999)
radiance = ht.HyTools()
radiance.read_file(rdn_file, 'envi')
#Average over Landsat 8 Band 5 bandwidth and warp
unwarp_band = np.zeros(longitude.shape)
for wave in range(850,890,10):
unwarp_band += radiance.get_wave(wave)/7.
warp_band = project.project_band(unwarp_band,-9999)
warp_band = 16000*(warp_band-warp_band.min())/warp_band.max()
if isinstance(match,bool):
landsat,land_east,land_north = get_landsat_image(longitude,latitude,
mean_time.month,
max_cloud = 5)
else:
lst = ht.HyTools()
lst.read_file(match,'envi')
landsat = lst.get_band(0)
land_east = float(lst.map_info[3])
land_north = float(lst.map_info[4])
#Calculate offsets between reference and input images
offset_x = int((warp_east-land_east)//pixel_size)
offset_y = int((land_north-warp_north)//pixel_size)
#Calculate optimal shift
y_model,x_model = image_match(landsat,warp_band,
offset_x,offset_y,
sensor_zn_prj,sensor_az_prj,elevation_prj)
#Apply uniform filter
smooth_elevation = uniform_filter(elevation,25)
smooth_az = uniform_filter(sensor_az,25)
smooth_zn = uniform_filter(sensor_zn,25)
# Generate y and x offset surfaces
i,a,b,c = y_model
y_offset = i + a*smooth_zn +b*smooth_az + c*smooth_elevation
i,a,b,c= x_model
x_offset = i + a*smooth_zn +b*smooth_az + c*smooth_elevation
# Calculate updated coordinates
easting = easting+ 30*x_offset
northing = northing- 30*y_offset
zone,direction = utm_zone(longitude,latitude)
longitude,latitude = utm2dd(easting,northing,zone,direction)
#Recalculate elevation with new coordinates
logging.info('Rebuilding DEM')
elevation= dem_generate(longitude,latitude,elev_dir,temp_dir)
# Export location datacube
loc_export(loc_file,longitude,latitude,elevation)
# Generate remaining observable layers
slope,aspect = slope_aspect(elevation,temp_dir)
cosine_i = calc_cosine_i(np.radians(solar_zn),
np.radians(solar_az),
np.radians(slope),
np.radians(aspect))
rel_az = np.radians(solar_az-sensor_az)
phase = np.arccos(np.cos(np.radians(solar_zn)))*np.cos(np.radians(solar_zn))
phase += np.sin(np.radians(solar_zn))*np.sin(np.radians(solar_zn))*np.cos(rel_az)
# Export observables datacube
obs_export(obs_file,path,sensor_az,sensor_zn,
solar_az,solar_zn,phase,slope,aspect,
cosine_i,utc_time)
if proj:
#Create new projector with corrected coordinates
new_coords =np.concatenate([np.expand_dims(easting.flatten(),axis=1),
np.expand_dims(northing.flatten(),axis=1)],axis=1)
project = Projector()
project.create_tree(new_coords,easting.shape)
project.query_tree(easting.min()-100,northing.max()+100,30)
blocksize = int(res/30)
map_info = ['UTM', 1, 1, easting.min()-100 - (res/2), northing.max()+100 + (res/2),res,
res,zone,direction, 'WGS-84' , 'units=Meters']
out_cols = int(blocksize* (project.output_shape[1]//blocksize))
out_lines = int(blocksize* (project.output_shape[0]//blocksize))
logging.info('Georeferencing datasets to %sm resolution' % res)
for file in ['rdn','loc','obs']:
input_name = '%sPRS_%s_%s' % (temp_dir,base_name,file)
hy_obj = ht.HyTools()
hy_obj.read_file(input_name, 'envi')
iterator =hy_obj.iterate(by = 'band')
out_header = hy_obj.get_header()
out_header['lines']= project.output_shape[0]//blocksize
out_header['samples']=project.output_shape[1]//blocksize
out_header['data ignore value'] = -9999
out_header['map info'] = map_info
output_name = '%sPRS_%s_%s_prj' % (out_dir,base_name,file)
writer = WriteENVI(output_name,out_header)
while not iterator.complete:
if (file == 'obs') & (iterator.current_band in [1,2,3,4,7]):
angular = True
else:
angular = False
band = project.project_band(iterator.read_next(),-9999,angular=angular)
band[band == -9999] = np.nan
bins =view_as_blocks(band[:out_lines,:out_cols], (blocksize,blocksize))
if angular:
bins = np.radians(bins)
band = circmean(bins,axis=2,nan_policy = 'omit')
band = circmean(band,axis=2,nan_policy = 'omit')
band = np.degrees(band)
else:
band = np.nanmean(bins,axis=(2,3))
if file == 'rdn':
band[band<0] = 0
band[np.isnan(band)] = -9999
writer.write_band(band,iterator.current_band)
logging.info('Deleting temporary files')
shutil.rmtree(temp_dir)
iso_base = '%s/isotest/isofit/' % home
figure_dir = '%s/sister/figures/prisma/radiance_factors/' % home
doy = dt.datetime.strptime(base_name[:8],'%Y%m%d').timetuple().tm_yday
base_file = '%s/data/prisma/rad/PRISMA_%s_rad_factor/PRISMA_%s_' % (home, base_name,base_name)
cal_dir = '%s/temp/%s' % (home,base_name)
if os.path.isdir(cal_dir):
shutil.rmtree(cal_dir)
os.mkdir(cal_dir)
for subdir in ['/configs','/data','/output','/lut_full/']:
os.mkdir(cal_dir+subdir)
#Load datasets
radiance = ht.HyTools()
radiance.read_file(base_file + 'rad_unprj', 'envi')
radiance.create_bad_bands([[300,380.0],[1300.0,1500],[1780.0,1975.0],[2450.0,2600]])
observables = ht.HyTools()
observables.read_file(base_file + 'obs_unprj', 'envi')
location = ht.HyTools()
location.read_file(base_file + 'loc_unprj', 'envi')
#Get center of image
obs_mean = observables.get_chunk(0,-1,0,-1).mean(axis =(0,1))
loc_mean = location.get_chunk(0,-1,0,-1).mean(axis =(0,1))
# Export wavelength file
wavelength_file = '%s/data/wavelengths.txt' % cal_dir
def resample(in_file, out_dir, resolution, verbose=True, unrotate=False):
''' Perform a two-step spatial resampling to . First, pixels are aggregated and
averaged, next a nearest neighbor algorithm is used to resample images to resolution.
'''
out_image = out_dir + '/' + os.path.basename(in_file)
image = ht.HyTools()
image.read_file(in_file, 'envi')
x_ul = float(image.map_info[3])
y_ul = float(image.map_info[4])
pixel_res = float(image.map_info[5])
if 'rotation' in image.map_info[-1]:
rotation = 360 + float(image.map_info[-1].split('=')[-1])
else:
rotation = 0
# Calculate rotated and unrotated coords
y_ind, x_ind = np.indices((image.lines, image.columns))
y_rcoord = y_ul - y_ind * pixel_res
x_rcoord = x_ul + x_ind * pixel_res
if unrotate:
x_coord, y_coord = rotate_coords(x_rcoord, y_rcoord, x_ul, y_ul,
rotation)
else:
x_coord, y_coord = x_rcoord, y_rcoord
bin_size = int(np.round(resolution / pixel_res))
if verbose:
print("Aggregating every %s pixels" % bin_size)
lines = bin_size * (image.lines // bin_size)
columns = bin_size * (image.columns // bin_size)
y_coord_bin = np.nanmean(view_as_blocks(y_coord[:lines, :columns],
(bin_size, bin_size)),
axis=(2, 3))
x_coord_bin = np.nanmean(view_as_blocks(x_coord[:lines, :columns],
(bin_size, bin_size)),
axis=(2, 3))
# Get extent of output array
xmax = int(resolution * (x_coord_bin.max() // resolution)) + resolution
ymax = int(resolution * (y_coord_bin.max() // resolution)) + resolution
xmin = int(resolution * (x_coord_bin.min() // resolution)) - resolution
ymin = int(resolution * (y_coord_bin.min() // resolution)) - resolution
out_columns = int((xmax - xmin) / resolution)
out_lines = int((ymax - ymin) / resolution)
#Calculate coordinates of output array
image_shape = (out_lines, out_columns)
y_coord_out, x_coord_out = np.indices(image_shape) * resolution
y_coord_out = ymax - y_coord_out
x_coord_out = xmin + x_coord_out
#Create tree to convert pixels to geolocated pixels
src_points = np.concatenate([
np.expand_dims(x_coord_bin.flatten(), axis=1),
np.expand_dims(y_coord_bin.flatten(), axis=1)
],
axis=1)
tree = cKDTree(src_points, balanced_tree=True)
dst_points = np.concatenate([
np.expand_dims(x_coord_out.flatten(), axis=1),
np.expand_dims(y_coord_out.flatten(), axis=1)
],
axis=1)
dists, indexes = tree.query(dst_points, k=1)
indices_int = np.unravel_index(indexes, x_coord_bin.shape)
mask = dists.reshape(image_shape) > resolution
out_header = image.get_header()
out_header['lines'] = out_lines
out_header['samples'] = out_columns
out_header['map info'][3] = str(xmin)
out_header['map info'][4] = str(ymax)
out_header['map info'][5:7] = resolution, resolution
if unrotate:
out_header['map info'][-1] = 'rotation=0.0000'
out_header['byte order'] = 0
out_header['data ignore value'] = image.no_data
writer = WriteENVI(out_image, out_header)
iterator = image.iterate(by='band')
while not iterator.complete:
if verbose & (iterator.current_band % 10 == 0):
print("%s/%s" % (iterator.current_band, image.bands))
band = np.copy(iterator.read_next()).astype(float)
band[~image.mask['no_data']] = np.nan
bins = view_as_blocks(band[:lines, :columns], (bin_size, bin_size))
if (iterator.current_band in [1, 2, 3, 4, 7]) and ('obs'
in image.base_name):
bins = np.radians(bins)
band = circmean(bins, axis=2, nan_policy='omit')
band = circmean(band, axis=2, nan_policy='omit')
band = np.degrees(band)
else:
band = np.nanmean(bins, axis=(2, 3))
band = band[indices_int[0], indices_int[1]].reshape(image_shape)
band[mask] = image.no_data
band[np.isnan(band)] = image.no_data
writer.write_band(band, iterator.current_band)
def dem_generate(longitude, latitude, elev_dir, temp_dir):
'''
Args:
longitude (float): Longitude array
latitude (float): Latitude array
elev_dir (str): Directory of zipped elevation tiles
temp_dir (str): Temporary output directory
Returns:
dem (np.array): Elevation array.
'''
# Get extents of image
lon_min = longitude.min()
lon_max = longitude.max()
lat_min = latitude.min()
lat_max = latitude.max()
if 'aws' in elev_dir:
tiles = pd.read_csv(elev_dir + 'tileList.txt',
header=None).values.flatten()
else:
tiles = glob.glob(elev_dir + '*.tar.gz')
idx = index.Index(properties=index.Property())
#Get list of intersecting tiles
for i, tile in enumerate(tiles):
lat, ign, lon = os.path.basename(tile).replace('_COG',
'').split('_')[3:6]
if 'W' in lon:
lon = -1 * float(lon[1:])
else:
lon = float(lon[1:])
if 'S' in lat:
lat = -1 * float(lat[1:])
else:
lat = float(lat[1:])
idx.insert(i, (lon, lat, lon + 1, lat + 1))
tiles_intersect = [
tiles[n]
for n in idx.intersection((lon_min, lat_min, lon_max, lat_max))
]
if len(tiles_intersect) == 0:
constant_elev = float(
input(
"No overlapping tiles found, enter constant elevation for scene (m): "
))
elevation = np.ones(longitude.shape) * constant_elev
else:
tile_string = "Found %s intersecting elevation tiles:" % len(
tiles_intersect)
for tile in tiles_intersect:
tile_string += '\n\t%s' % tile
if 'aws' in elev_dir:
tile_url = "%s%s/%s.tif" % (elev_dir, tile, tile)
tile_file = "%s%s.tif" % (temp_dir, tile)
download_file(tile_file, tile_url)
else:
with tarfile.open(tile, 'r') as tar_ref:
tar_ref.extractall(temp_dir)
logging.info(tile_string)
logging.info('Merging DEM tiles')
dem_file = '%stemp_dem' % temp_dir
os.system('gdal_merge.py -o %s -of ENVI %sCopernicus_DSM*' %
(dem_file, temp_dir))
dem_obj = ht.HyTools()
dem_obj.read_file(dem_file, 'envi')
ulx = float(dem_obj.map_info[3])
uly = float(dem_obj.map_info[4])
pix = float(dem_obj.map_info[5])
dem_lat, dem_lon = np.indices((dem_obj.lines, dem_obj.columns))
dem_xl = int((lon_min - ulx) // pix)
dem_xr = int((lon_max - ulx) // pix)
dem_yu = int((uly - lat_max) // pix)
dem_yd = int((uly - lat_min) // pix)
dem_subset = dem_obj.get_chunk(dem_xl, dem_xr, dem_yu, dem_yd)
dem_lat, dem_lon = np.indices(dem_subset.shape[:2])
dem_lat = (lat_max - dem_lat * pix).flatten()
dem_lon = (lon_min + dem_lon * pix).flatten()
#Create spatial index and nearest neighbor sample
src_points = np.concatenate(
[np.expand_dims(dem_lon, axis=1),
np.expand_dims(dem_lat, axis=1)],
axis=1)
tree = cKDTree(src_points, balanced_tree=False)
dst_points = np.concatenate([
longitude.flatten()[:, np.newaxis],
latitude.flatten()[:, np.newaxis]
],
axis=1)
indexes = tree.query(dst_points, k=1)[1]
indices_int = np.unravel_index(indexes,
(dem_subset.shape[0], dem_subset.shape[1]))
elevation = dem_subset[indices_int[0],
indices_int[1]].reshape(longitude.shape)
#Set negative elevations to 0
if np.sum(elevation < 0) > 0:
logging.warning('Elevations below sea level found, setting to 0m')
elevation[elevation < 0] = 0
return elevation
def l1b_process(l1b_zip,
out_dir,
temp_dir,
elev_dir,
match=None,
proj=True,
res=30):
'''
This function exports three files:
*_rad* : Merged and optionally shift corrected radiance cube
*_obs* : Observables file in the format of JPL obs files:
1. Pathlength (m)
2. To-sensor view azimuth angle (degrees)
3. To-sensor view zenith angle (degrees)
4. To-sun azimuth angle (degrees)
5. To-sun zenith angle (degrees)
6. Phase
7. Slope (Degrees)
8. Aspect (Degrees)
9. Cosine i
10. UTC decimal hours
*_loc* : Location file in the following format:
1. Longitude (decimal degrees)
2. Longitude (decimal degrees)
3. Elevation (m)
l1(str): L1B zipped radiance data product path
out_dir(str): Output directory of ENVI datasets
temp_dir(str): Temporary directory for intermediate
elev_dir (str): Directory zipped Copernicus elevation tiles or url to AWS Copernicus data
ex : 'https://copernicus-dem-30m.s3.amazonaws.com/'
match (str or list) : Pathname to Landsat image for image re-registration (recommended)
proj (bool) : Project image to UTM grid
res (int) : Resolution of projected image, 30 should be one of its factors (90,120,150.....)
'''
base_name = os.path.basename(l1b_zip)[14:-4]
out_dir = '%s/DESIS_%s/' % (out_dir, base_name)
if not os.path.isdir(out_dir):
os.mkdir(out_dir)
temp_dir = '%s/tmpDESIS_%s/' % (temp_dir, base_name)
if not os.path.isdir(temp_dir):
os.mkdir(temp_dir)
zip_base = os.path.basename(l1b_zip)
logging.info('Unzipping %s' % zip_base)
with zipfile.ZipFile(l1b_zip, 'r') as zipped:
zipped.extractall(temp_dir)
l1b_file = gdal.Open('%s/DESIS-HSI-L1B-%s-SPECTRAL_IMAGE.tif' %
(temp_dir, base_name))
# Parse relevant metadata from XML file, assume metadata are in same directory as iamges
tree = ET.parse('%s/DESIS-HSI-L1B-%s-METADATA.xml' % (temp_dir, base_name))
root = tree.getroot()
specific = root[3]
band_meta = {}
for item in specific.findall('bandCharacterisation'):
for band in item:
for meta in band:
if meta.tag not in band_meta.keys():
band_meta[meta.tag] = []
string = str(meta.text.encode('utf8'))
string = string.replace('\\n', ' ')
string = string.replace('b', ' ')
string = string.replace("'", ' ')
values = string.split(',')
values = [float(x) for x in values]
if len(values) == 1:
values = values[0]
band_meta[meta.tag].append(values)
offset = np.array(band_meta['offsetOfBand'])
gain = np.array(band_meta['gainOfBand'])
waves = np.array(band_meta['wavelengthCenterOfBand'])
fwhm = np.array(band_meta['wavelengthWidthOfBand'])
response = np.array(band_meta['response'])
response_waves = np.array(band_meta['wavelengths'])
# Fit a gaussian to the reponse to determine center wavelength and fhwm
opt_waves = []
opt_fwhm = []
for i, wave in enumerate(waves):
popt, pcov = curve_fit(gaussian, response_waves[i],
np.array(response[i]) / max(response[i]),
[waves[i], fwhm[i]])
opt_waves.append(popt[0])
opt_fwhm.append(popt[1])
scene_az = float(specific.findall('sceneAzimuthAngle')[0].text)
scene_zn = float(specific.findall('sceneIncidenceAngle')[0].text)
# Get acquisition start and end time
base = root[2]
time_str = base.findall('temporalCoverage')[0].findall('startTime')[0].text
time_str = time_str.replace('T', ' ').replace('Z', '')
start_time = dt.datetime.strptime(time_str, "%Y-%m-%d %H:%M:%S.%f")
start_time = start_time.replace(tzinfo=dt.timezone.utc)
time_str = base.findall('temporalCoverage')[0].findall('endTime')[0].text
time_str = time_str.replace('T', ' ').replace('Z', '')
end_time = dt.datetime.strptime(time_str, "%Y-%m-%d %H:%M:%S.%f")
end_time = end_time.replace(tzinfo=dt.timezone.utc)
date = dt.datetime.strftime(start_time, "%Y%m%d")
# Get orbital data
orbit_data = []
for item in specific.findall('orbit'):
for line in item:
for point in line.findall('point'):
time_str = line.findall('timeUTC')[0].text
time_str = time_str.replace('T', ' ').replace('Z', '')
orbit_time = dt.datetime.strptime(time_str,
"%Y-%m-%d %H:%M:%S.%f")
orbit_time = orbit_time.replace(tzinfo=dt.timezone.utc)
#The metadata contains orbital info beyond the collection window
# we use the acquisition start and end times to filter points
if (orbit_time >= start_time) & (orbit_time <= end_time):
for location in point.findall('location'):
x = float(location.findall('X')[0].text)
y = float(location.findall('Y')[0].text)
z = float(location.findall('Z')[0].text)
orbit_data.append([x, y, z])
orbit_data = np.array(orbit_data)
l1b_band = l1b_file.ReadAsArray().mean(axis=0)
# Get bounding coordinates of scene
coord_dict = {}
polygon = base.findall('spatialCoverage')[0].findall('boundingPolygon')[0]
for point in polygon:
name = point.findall('frame')[0].text
lat = float(point.findall('latitude')[0].text)
lon = float(point.findall('longitude')[0].text)
coord_dict[name] = [lat, lon]
# Get ISS altitude
altitude_m = float(base.findall('altitudeCoverage')[0].text)
raster = l1b_file.ReadAsArray()
mask = raster[1].astype(float)
mask = mask == mask[0][0]
rad_dict = envi_header_dict()
rad_dict['lines'] = l1b_file.RasterYSize
rad_dict['samples'] = l1b_file.RasterXSize - 85
rad_dict['bands'] = len(waves) - 1
rad_dict['wavelength'] = opt_waves[1:]
rad_dict['fwhm'] = opt_fwhm[1:]
rad_dict['interleave'] = 'bil'
rad_dict['data type'] = 4
rad_dict['wavelength units'] = "nanometers"
rad_dict['byte order'] = 0
rad_dict['data ignore value'] = -9999
rad_dict['default bands'] = [
np.argmin(np.abs(waves - 660)),
np.argmin(np.abs(waves - 560)),
np.argmin(np.abs(waves - 460))
]
#Define output paths
if proj:
rad_file = '%sDESIS_%s_rdn' % (temp_dir, base_name)
loc_file = '%sDESIS_%s_loc' % (temp_dir, base_name)
obs_file = '%sDESIS_%s_obs' % (temp_dir, base_name)
else:
rad_file = '%sDESIS_%s_rdn' % (out_dir, base_name)
loc_file = '%sDESIS_%s_loc' % (out_dir, base_name)
obs_file = '%sDESIS_%s_obs' % (out_dir, base_name)
writer = WriteENVI(rad_file, rad_dict)
#Write VNIR cube
logging.info('Exporting radiance data')
for line_num in range(l1b_file.RasterYSize):
line = raster[:, line_num, :].astype(float)
line = line * gain[:, np.newaxis] + offset[:, np.newaxis]
line = line[1:, 85:].T
writer.write_line(line, line_num)
del raster
# Location datacube
###########################################################################
lines, columns = np.indices((l1b_file.RasterYSize, l1b_file.RasterXSize))
lat_vals = []
lon_vals = []
points = [[0, 0], [l1b_file.RasterYSize, 0],
[l1b_file.RasterYSize, l1b_file.RasterXSize],
[0, l1b_file.RasterXSize]]
for point in [1, 2, 3, 4]:
lat_vals.append(coord_dict['point_%s' % point][0])
lon_vals.append(coord_dict['point_%s' % point][1])
longitude = griddata(points, lon_vals, (lines, columns),
method='linear')[:, 85:]
latitude = griddata(points, lat_vals, (lines, columns),
method='linear')[:, 85:]
#Create initial elevation raster
elevation = dem_generate(longitude, latitude, elev_dir, temp_dir)
zone, direction = utm_zone(longitude, latitude)
solar_az = solar.get_azimuth(latitude, longitude, start_time)
solar_zn = 90 - solar.get_altitude(latitude, longitude, start_time)
ecef = pyproj.Proj(proj='geocent', ellps='WGS84', datum='WGS84')
lla = pyproj.Proj(proj='latlong', ellps='WGS84', datum='WGS84')
grd_xyz = np.array(
pyproj.transform(lla,
ecef,
longitude,
latitude,
elevation,
radians=False))
# Calculate satellite XYZ position
sat_xyz = []
line_grid = np.linspace(0, 1, orbit_data.shape[0]) * longitude.shape[0]
for sat_coord in orbit_data.T:
interpolator = interp1d(line_grid, sat_coord)
sat_interp = interpolator(np.arange(longitude.shape[0]))
sat_xyz.append(sat_interp)
sat_xyz = np.array(sat_xyz)
path = np.linalg.norm(sat_xyz[:, :, np.newaxis] - grd_xyz, axis=0)
# Export satellite position to csv
sat_lon, sat_lat, sat_alt = ecef2dda(sat_xyz[0], sat_xyz[1], sat_xyz[2])
satellite_df = pd.DataFrame()
satellite_df['lat'] = sat_lat
satellite_df['lon'] = sat_lon
satellite_df['alt'] = sat_alt
satellite_df.to_csv('%sDESIS_%s_satellite_loc.csv' % (out_dir, base_name))
# Convert satellite coords to local ENU
sat_enu = np.array(
dda2utm(sat_lon, sat_lat, sat_alt, utm_zone(longitude, latitude)))
# Convert ground coords to local ENU
easting, northing, up = dda2utm(longitude, latitude, elevation)
# Calculate sensor geometry
sensor_zn, sensor_az = sensor_view_angles(
sat_enu, np.array([easting, northing, up]))
if match:
coords = np.concatenate([
np.expand_dims(easting.flatten(), axis=1),
np.expand_dims(northing.flatten(), axis=1)
],
axis=1)
warp_east = easting.min() - 100
warp_north = northing.max() + 100
pixel_size = 30
project = Projector()
project.create_tree(coords, easting.shape)
project.query_tree(warp_east, warp_north, pixel_size)
# Project independent variables
sensor_az_prj = project.project_band(sensor_az, -9999)
sensor_zn_prj = project.project_band(sensor_zn, -9999)
elevation_prj = project.project_band(elevation.astype(np.float), -9999)
radiance = ht.HyTools()
radiance.read_file(rad_file, 'envi')
#Average over Landsat 8 Band 5 bandwidth and warp
warp_band = np.zeros(longitude.shape)
for wave in range(850, 890, 10):
warp_band += radiance.get_wave(wave) / 7.
warp_band = project.project_band(warp_band, -9999)
warp_band = 16000 * (warp_band - warp_band.min()) / warp_band.max()
landsat, land_east, land_north = get_landsat_image(longitude,
latitude,
end_time.month,
max_cloud=5)
#Calculate offsets between reference and input images
offset_x = int((warp_east - land_east) // pixel_size)
offset_y = int((land_north - warp_north) // pixel_size)
#Calculate optimal shift
y_model, x_model = image_match(landsat,
warp_band,
offset_x,
offset_y,
sensor_zn_prj,
sensor_az_prj,
elevation_prj,
shift_max=30)
#Apply uniform filter
smooth_elevation = uniform_filter(elevation, 25)
smooth_az = uniform_filter(sensor_az, 25)
smooth_zn = uniform_filter(sensor_zn, 25)
# Generate y and x offset surfaces
i, a, b, c = y_model
y_offset = i + a * smooth_zn + b * smooth_az + c * smooth_elevation
i, a, b, c = x_model
x_offset = i + a * smooth_zn + b * smooth_az + c * smooth_elevation
# Calculate updated coordinates
easting = easting + 30 * x_offset
northing = northing - 30 * y_offset
zone, direction = utm_zone(longitude, latitude)
longitude, latitude = utm2dd(easting, northing, zone, direction)
#Recalculate elevation with new coordinates
logging.info('Rebuilding DEM')
elevation = dem_generate(longitude, latitude, elev_dir, temp_dir)
loc_export(loc_file, longitude, latitude, elevation)
# Generate remaining observable layers
slope, aspect = slope_aspect(elevation, temp_dir)
cosine_i = calc_cosine_i(np.radians(solar_zn), np.radians(solar_az),
np.radians(slope), np.radians(aspect))
rel_az = np.radians(solar_az - sensor_az)
phase = np.arccos(np.cos(np.radians(solar_zn))) * np.cos(
np.radians(solar_zn))
phase += np.sin(np.radians(solar_zn)) * np.sin(
np.radians(solar_zn)) * np.cos(rel_az)
utc_time = (lines / (l1b_file.RasterYSize) *
(end_time - start_time).seconds) / 60 / 60
utc_time += start_time.hour + start_time.minute / 60
utc_time = utc_time[:, 85:]
obs_export(obs_file, path, sensor_az, sensor_zn, solar_az, solar_zn, phase,
slope, aspect, cosine_i, utc_time)
if proj:
#Create new projector with corrected coordinates
new_coords = np.concatenate([
np.expand_dims(easting.flatten(), axis=1),
np.expand_dims(northing.flatten(), axis=1)
],
axis=1)
project = Projector()
project.create_tree(new_coords, easting.shape)
project.query_tree(easting.min() - 100, northing.max() + 100, 30)
blocksize = int(res / 30)
map_info = [
'UTM', 1, 1,
easting.min() - 100,
northing.max() + 100, res, res, zone, direction, 'WGS-84',
'units=Meters'
]
out_cols = int(blocksize * (project.output_shape[1] // blocksize))
out_lines = int(blocksize * (project.output_shape[0] // blocksize))
logging.info('Georeferencing datasets')
for file in ['rdn', 'loc', 'obs']:
logging.info(file)
input_name = '%sDESIS_%s_%s' % (temp_dir, base_name, file)
hy_obj = ht.HyTools()
hy_obj.read_file(input_name, 'envi')
iterator = hy_obj.iterate(by='band')
out_header = hy_obj.get_header()
out_header['lines'] = project.output_shape[0] // blocksize
out_header['samples'] = project.output_shape[1] // blocksize
out_header['data ignore value'] = -9999
out_header['map info'] = map_info
output_name = '%sDESIS_%s_%s_prj' % (out_dir, base_name, file)
writer = WriteENVI(output_name, out_header)
while not iterator.complete:
band = project.project_band(iterator.read_next(), -9999)
band[band == -9999] = np.nan
band = np.nanmean(view_as_blocks(band[:out_lines, :out_cols],
(blocksize, blocksize)),
axis=(2, 3))
if file == 'rdn':
band[band < 0] = 0
band[np.isnan(band)] = -9999
writer.write_band(band, iterator.current_band)
logging.info('Deleting temporary files')
shutil.rmtree(temp_dir)
def h5_radiance_to_envi(filename, resolution=1):
'''Convert a NEON HDF radiance file to ENVI formated
image along with observables and location data cubes
TODO: Recalculate terrain azimuth, provided product may be
incorrect
Args:
filename (str): Path to HDF file.
resolution (int, optional): Output image resolution. Defaults to 1.
Returns:
None.
'''
# Load HDF file
hdf_obj = h5py.File(filename, 'r')
key = [key for key in hdf_obj.keys()][0]
rad_dec = hdf_obj[key]['Radiance']['RadianceDecimalPart']
rad_int = hdf_obj[key]['Radiance']['RadianceIntegerPart']
obs = hdf_obj[key]['Radiance']['Metadata']['Ancillary_Rasters']['OBS_Data']
igm = hdf_obj[key]['Radiance']['Metadata']['Ancillary_Rasters']['IGM_Data']
wavelengths = hdf_obj[key]['Radiance']['Metadata']['Spectral_Data'][
'Wavelength'][:].tolist()
fwhm = hdf_obj[key]['Radiance']['Metadata']['Spectral_Data'][
'FWHM'][:].tolist()
map_info = hdf_obj[key]['Radiance']['Metadata']['Coordinate_System'][
'Map_Info'][()].decode("utf-8").split(',')
epsg = hdf_obj[key]['Radiance']['Metadata']['Coordinate_System'][
'EPSG Code'][()].decode("utf-8")
new_lines = rad_dec.shape[0] // resolution
new_cols = rad_dec.shape[1] // resolution
map_info[5] = resolution
map_info[6] = resolution
map_info = [str(info).strip() for info in map_info]
# Export integer and decimal radiance components
# to temporary ENVI files
rad_dict = envi_header_dict()
rad_dict['lines'] = rad_dec.shape[0]
rad_dict['samples'] = rad_dec.shape[1]
rad_dict['bands'] = rad_dec.shape[2]
rad_dict['interleave'] = 'bsq'
rad_dict['data type'] = 12
rad_dict['byte order'] = 0
dec_temp = filename.replace('radiance.h5', 'rad_dec')
writer = WriteENVI(dec_temp, rad_dict)
writer.write_chunk(rad_dec, 0, 0)
int_temp = filename.replace('radiance.h5', 'rad_int')
writer = WriteENVI(int_temp, rad_dict)
writer.write_chunk(rad_int, 0, 0)
int_obj = ht.HyTools()
int_obj.read_file(int_temp, 'envi')
dec_obj = ht.HyTools()
dec_obj.read_file(dec_temp, 'envi')
# Export radiance
##################
rad_dict = envi_header_dict()
rad_dict['lines'] = new_lines
rad_dict['samples'] = new_cols
rad_dict['bands'] = rad_dec.shape[2]
rad_dict['wavelength'] = wavelengths
rad_dict['fwhm'] = fwhm
rad_dict['interleave'] = 'bil'
rad_dict['data type'] = 4
rad_dict['wavelength units'] = "nanometers"
rad_dict['byte order'] = 0
rad_dict['data ignore value'] = -9999
rad_dict['map info'] = map_info
output_name = filename.replace('radiance.h5', 'rad')
writer = WriteENVI(output_name, rad_dict)
for band_num in range(rad_dict['bands']):
print(band_num)
band_int = int_obj.get_band(band_num).astype(float)
band_dec = dec_obj.get_band(band_num) / 50000
band = band_int + band_dec
band[band_int == 255] = np.nan
band = band[:new_lines * resolution, :new_cols * resolution]
band = view_as_blocks(band, (resolution, resolution)).mean(axis=(2, 3))
band[np.isnan(band)] = -9999
writer.write_band(band, band_num)
os.remove(dec_temp)
os.remove(int_temp)
# Export observables
####################
obs_dict = envi_header_dict()
obs_dict['band_names'] = [
'path length', 'to-sensor azimuth', 'to-sensor zenith',
'to-sun azimuth', 'to-sun zenith', 'phase', 'slope', 'aspect',
'cosine i', 'UTC time'
]
obs_dict['data type'] = 4
obs_dict['lines'] = new_lines
obs_dict['samples'] = new_cols
obs_dict['bands'] = 10
obs_dict['fwhm'] = fwhm
obs_dict['interleave'] = 'bil'
obs_dict['data type'] = 4
obs_dict['byte order'] = 0
obs_dict['data ignore value'] = -9999
obs_dict['map info'] = map_info
output_name = filename.replace('radiance.h5', 'obs')
writer = WriteENVI(output_name, obs_dict)
for band_num in range(obs_dict['bands']):
print(band_num)
band = obs[:, :, band_num]
band[band == -9999] = np.nan
band = band[:new_lines * resolution, :new_cols * resolution]
band = view_as_blocks(band, (resolution, resolution)).mean(axis=(2, 3))
band[np.isnan(band)] = -9999
writer.write_band(band, band_num)
# Export location datacube (lon,lat,elevation)
##############################################
loc_dict = envi_header_dict()
loc_dict['band_names'] = ['longitude', 'latitude', 'elevation']
loc_dict['data type'] = 4
loc_dict['lines'] = new_lines
loc_dict['samples'] = new_cols
loc_dict['bands'] = 3
loc_dict['fwhm'] = fwhm
loc_dict['interleave'] = 'bil'
loc_dict['data type'] = 4
loc_dict['byte order'] = 0
loc_dict['data ignore value'] = -9999
loc_dict['map info'] = map_info
output_name = filename.replace('radiance.h5', 'loc')
writer = WriteENVI(output_name, loc_dict)
in_proj = pyproj.Proj("+init=EPSG:%s" % epsg)
out_proj = pyproj.Proj("+init=EPSG:4326")
longitude, latitude = pyproj.transform(in_proj, out_proj, igm[:, :, 0],
igm[:, :, 1])
elevation = igm[:, :, 2]
mask = elevation == -9999
for band_num, band in enumerate([longitude, latitude, elevation]):
print(band_num)
band[mask] = np.nan
band = band[:new_lines * resolution, :new_cols * resolution]
band = view_as_blocks(band, (resolution, resolution)).mean(axis=(2, 3))
band[np.isnan(band)] = -9999
writer.write_band(band, band_num)
os.remove(filename)
def reflectance(self):
'''Run ISOFIT reflectance inversion
'''
# Check for input files
if not (self.exists(self.rdn_file) & self.exists(self.loc_file) & self.exists(self.obs_file)):
print('Radiance data not found, generating radiance datasets')
self.radiance()
rfl_tmp = self.tmp_dir + '%s_isofit/' % self.base_name
if not os.path.isdir(rfl_tmp):
os.mkdir(rfl_tmp)
else:
shutil.rmtree(rfl_tmp)
os.mkdir(rfl_tmp)
for key in self.isofit['surface']:
get_surface_spectra(rfl_tmp + key,
self.isofit['surface'][key])
surface_config = "%s/surface_config.json" % rfl_tmp
surface_file = '%s/surface_filtered.mat'% rfl_tmp
wavelength_file = gen_wavelength_file(self.rdn_file,
rfl_tmp)
surface_config_gen(rfl_tmp,
self.isofit['surface_type'],
self.isofit['windows'],
wavelength_file,
surface_file,
surface_config)
surface_model(surface_config)
apply_oe = ['python']
apply_oe.append('%s/isofit/utils/apply_oe.py' % self.isofit['base'])
apply_oe.append(self.rdn_file)
apply_oe.append(self.loc_file)
apply_oe.append(self.obs_file)
apply_oe.append(rfl_tmp)
if self.sensor in ['prisma','desis']:
apply_oe.append('NA-%s' % self.date)
apply_oe += ['--surface_path', surface_file]
apply_oe += ['--n_cores',str(cpu_count())]
apply_oe += ['--empirical_line',str(self.isofit['empirical_line'])]
apply_oe += ['--presolve',str(self.isofit['presolve'])]
apply_oe += ['--ray_temp_dir',rfl_tmp]
apply_oe += ['--log_file','%s/%s_logfile' % (rfl_tmp,self.base_name)]
apply_oe += ['--emulator_base',self.isofit['emulator']]
if self.rdn_cfg['radiance_factors']:
rad_factors = "%s/radiance_factors.txt" % rfl_tmp
download_file(rad_factors,self.rdn_cfg['radiance_factors'])
apply_oe += ['--rdn_factors_path',rad_factors]
apply = subprocess.Popen(apply_oe)
apply.wait()
if not os.path.isdir(os.path.dirname(self.rfl_file)):
os.mkdir(os.path.dirname(self.rfl_file))
# Mask windows and rename ISOFIT output files
rfl = glob.glob('%s/output/*_rfl_prj_rfl' % rfl_tmp)[0]
uncert = glob.glob('%s/output/*_uncert_prj_uncert'% rfl_tmp)[0]
for new,old in [(self.rfl_file,rfl),(self.unc_file,uncert)]:
hy_obj = ht.HyTools()
hy_obj.read_file(old,'envi')
mask = []
for wave in hy_obj.wavelengths:
window = False
for start,end in self.isofit['windows']:
if (wave>start) and (wave<end):
window |= True
mask.append(window)
header_dict = hy_obj.get_header()
writer = ht.io.WriteENVI(new, header_dict)
for band_num in range(hy_obj.bands):
band = hy_obj.get_band(band_num)
if not mask[band_num]:
band = np.zeros(band.shape)
writer.write_band(band,band_num)
shutil.rmtree(rfl_tmp)
def preprocess(input_tar,out_dir,temp_dir,res = 0):
'''
input_tar = '/data2/avcl/raw/f080709t01p00r15.tar.gz'
input_tar = '/data2/avng/rdn/ang20170901t195659.tar.gz'
out_dir ='/data1/temp/ang_pre/output/'
temp_dir ='/data1/temp/ang_pre/temp/'
'''
base_name = os.path.basename(input_tar)
if base_name.startswith('ang'):
base_name = base_name[:18]
elif base_name.startswith('f'):
base_name = base_name[:16]
else:
raise ValueError('Unrecognized sensor')
out_dir = "%s/%s/" % (out_dir,base_name)
if not os.path.isdir(out_dir):
os.mkdir(out_dir)
file = tarfile.open(input_tar)
tar_contents = [temp_dir+c for c in file.getnames()]
file.extractall(temp_dir)
file.close()
#AVIRIS NG
if base_name.startswith('ang'):
obs_ort_file = [x for x in tar_contents if x.endswith('obs_ort')][0]
rdn_file = [x for x in tar_contents if x.endswith('img')][0]
loc_file = [x for x in tar_contents if x.endswith('loc')][0]
glt_file = [x for x in tar_contents if x.endswith('glt')][0]
create_loc_ort(loc_file,glt_file)
time_correct(obs_ort_file)
#AVIRIS Classic
elif base_name.startswith('f'):
obs_ort_file = [x for x in tar_contents if x.endswith('obs_ort')][0]
rdn_file = [x for x in tar_contents if x.endswith('ort_img')][0]
igm_file = [x for x in tar_contents if x.endswith('igm')][0]
glt_file = [x for x in tar_contents if x.endswith('ort_glt')][0]
#Rename files for consistency
loc_file = igm_file.replace('igm','loc')
os.rename(igm_file,loc_file)
os.rename(igm_file+'.hdr',loc_file+'.hdr')
#Scale radiance
os.rename(rdn_file,rdn_file+'_unscale')
os.rename(rdn_file + '.hdr',rdn_file+'_unscale.hdr')
rdn = ht.HyTools()
rdn.read_file(rdn_file+'_unscale','envi')
prefix = rdn.base_name[:3]
if prefix in ['f95', 'f96', 'f97', 'f98', 'f99', 'f00',
'f01', 'f02', 'f03', 'f04', 'f05']:
gains = np.r_[50.0 * np.ones(160), 100.0 * np.ones(64)]
elif prefix in ['f06', 'f07', 'f08', 'f09', 'f10', 'f11',
'f12', 'f13', 'f14', 'f15', 'f16', 'f17',
'f18', 'f19', 'f20', 'f21']:
gains = np.r_[300.0 * np.ones(110), 600.0 * np.ones(50), 1200.0 * np.ones(64)]
rdn_header = rdn.get_header()
rdn_header['byte order'] = 0
rdn_header['no data value'] = -9999
rdn_header['data type'] = 4
writer = WriteENVI(rdn.file_name.replace('_unscale',''),rdn_header)
iterator =rdn.iterate(by = 'band')
while not iterator.complete:
band = np.copy(iterator.read_next()).astype(float)
band /= gains[iterator.current_band]
band[~rdn.mask['no_data']] = -9999
writer.write_band(band,iterator.current_band)
create_loc_ort(loc_file,glt_file)
loc_ort_file = loc_file+'_ort'
for file in [obs_ort_file,rdn_file,loc_ort_file]:
if res==0:
shutil.move(file,out_dir)
shutil.move(file + '.hdr',out_dir)
else:
resample(file,out_dir,res)
shutil.rmtree(tar_contents[0])
