def map_ind(mod_lon, mod_lat, meas_lon, meas_lat, meas_good=None):
""" Run to get indices in the measurement space of all the closest mod points. Assuming earth geometry."""
from map_utils import spherical_dist
from Sp_parameters import startprogress, progress, endprogress
import numpy as np
try:
if not meas_good:
meas_good = np.where(meas_lon)
except ValueError:
if not meas_good.any():
meas_good = np.where(meas_lon)
imodis = np.logical_and(
np.logical_and(mod_lon > min(meas_lon[meas_good]) - 0.02,
mod_lon < max(meas_lon[meas_good]) + 0.02),
np.logical_and(mod_lat > min(meas_lat[meas_good]) - 0.02,
mod_lat < max(meas_lat[meas_good]) + 0.02))
wimodis = np.where(imodis)
if not wimodis[0].any():
print '** No points found within range +/- 0.02 in lat and lon, Extending range to +/- 0.2 **'
imodis = np.logical_and(
np.logical_and(mod_lon > min(meas_lon[meas_good]) - 0.2,
mod_lon < max(meas_lon[meas_good]) + 0.2),
np.logical_and(mod_lat > min(meas_lat[meas_good]) - 0.2,
mod_lat < max(meas_lat[meas_good]) + 0.2))
wimodis = np.where(imodis)
if not wimodis[0].any():
print '** No points found in extended range, returning null **'
return []
N1 = mod_lon[imodis].size
modis_grid = np.hstack(
[mod_lon[imodis].reshape((N1, 1)), mod_lat[imodis].reshape((N1, 1))])
try:
N2 = len(meas_good)
if N2 == 1 or N2 == 2:
meas_good = meas_good[0]
N2 = len(meas_good)
meas_grid = np.hstack([
np.array(meas_lon[meas_good]).reshape((N2, 1)),
np.array(meas_lat[meas_good]).reshape((N2, 1))
])
except:
import pdb
pdb.set_trace()
meas_in = meas_grid.astype(int)
meas_ind = np.array([meas_good.ravel() * 0, meas_good.ravel() * 0])
startprogress('Running through flight track')
for i in xrange(meas_good.size):
d = spherical_dist(meas_grid[i], modis_grid)
try:
meas_ind[0, i] = wimodis[0][np.argmin(d)]
except:
import pdb
pdb.set_trace()
meas_ind[1, i] = wimodis[1][np.argmin(d)]
progress(float(i) / len(meas_good) * 100)
endprogress()
return meas_ind
def map_ind(mod_lon,mod_lat,meas_lon,meas_lat,meas_good=None):
""" Run to get indices in the measurement space of all the closest mod points. Assuming earth geometry."""
from map_utils import spherical_dist
from Sp_parameters import startprogress, progress, endprogress
import numpy as np
try:
if not meas_good:
meas_good = np.where(meas_lon)
except ValueError:
if not meas_good.any():
meas_good = np.where(meas_lon)
imodis = np.logical_and(np.logical_and(mod_lon>min(meas_lon[meas_good])-0.02 , mod_lon<max(meas_lon[meas_good])+0.02),
np.logical_and(mod_lat>min(meas_lat[meas_good])-0.02 , mod_lat<max(meas_lat[meas_good])+0.02))
wimodis = np.where(imodis)
if not wimodis[0].any():
print '** No points found within range +/- 0.02 in lat and lon, Extending range to +/- 0.2 **'
imodis = np.logical_and(np.logical_and(mod_lon>min(meas_lon[meas_good])-0.2 , mod_lon<max(meas_lon[meas_good])+0.2),
np.logical_and(mod_lat>min(meas_lat[meas_good])-0.2 , mod_lat<max(meas_lat[meas_good])+0.2))
wimodis = np.where(imodis)
if not wimodis[0].any():
print '** No points found in extended range, returning null **'
return []
N1 = mod_lon[imodis].size
modis_grid = np.hstack([mod_lon[imodis].reshape((N1,1)),mod_lat[imodis].reshape((N1,1))])
try:
N2 = len(meas_good)
if N2==1 or N2==2:
meas_good = meas_good[0]
N2 = len(meas_good)
meas_grid = np.hstack([np.array(meas_lon[meas_good]).reshape((N2,1)),np.array(meas_lat[meas_good]).reshape((N2,1))])
except:
import pdb; pdb.set_trace()
meas_in = meas_grid.astype(int)
meas_ind = np.array([meas_good.ravel()*0,meas_good.ravel()*0])
startprogress('Running through flight track')
for i in xrange(meas_good.size):
d = spherical_dist(meas_grid[i],modis_grid)
try:
meas_ind[0,i] = wimodis[0][np.argmin(d)]
except:
import pdb; pdb.set_trace()
meas_ind[1,i] = wimodis[1][np.argmin(d)]
progress(float(i)/len(meas_good)*100)
endprogress()
return meas_ind
def load_hdf(datfile, values=None, verbose=True):
"""
Name:
load_hdf
Purpose:
to compile functions required to load emas files
from within another script.
Similar to load_modis
Calling Sequence:
hdf_dat,hdf_dict = load_hdf(datfile,Values=None,verbose=True)
Input:
datfile name (hdf files)
Output:
hdf_dat dictionary with the names of values saved, with associated dictionary values
hdf_dicts : metadate for each of the variables
Keywords:
values: if ommitted, only outputs the names of the variables in file
needs to be a tuple of 2 element tuples (first element name of variable, second number of record)
example: modis_values=(('cloud_top',57),('phase',53),('cloud_top_temp',58),('ref',66),('tau',72))
verbose: if true (default), then everything is printed. if false, nothing is printed
Dependencies:
gdal
numpy
gc: for clearing the garbage
Sp_parameters for progress issuer
Required files:
dat files
Example:
...
Modification History:
Written (v1.0): Samuel LeBlanc, 2014-12-10, NASA Ames
Modified (v1.1): Samuel LeBlanc, 2015-04-10, NASA Ames
- added verbose keyword
"""
import numpy as np
from osgeo import gdal
from Sp_parameters import startprogress, progress, endprogress
datsds = gdal.Open(datfile)
datsub = datsds.GetSubDatasets()
if verbose:
print 'Outputting the Data subdatasets:'
for i in range(len(datsub)):
if values:
if any(i in val for val in values):
print '\x1b[1;36m%i: %s\x1b[0m' % (i, datsub[i][1])
else:
print str(i) + ': ' + datsub[i][1]
else:
print str(i) + ': ' + datsub[i][1]
if not values:
if verbose:
print 'Done going through file... Please supply pairs of name and index for reading file'
print " in format values = (('name1',index1),('name2',index2),('name3',index3),...)"
print " where namei is the nameof the returned variable, and indexi is the index of the variable (from above)"
return None, None
hdf = dict()
meta = datsds.GetMetadata()
import gc
gc.collect()
hdf_dicts = dict()
if verbose:
startprogress('Running through data values')
for i, j in values:
sds = gdal.Open(datsub[j][0])
hdf_dicts[i] = sds.GetMetadata()
hdf[i] = np.array(sds.ReadAsArray())
if not hdf[i].any():
import pdb
pdb.set_trace()
try:
bad_points = np.where(hdf[i] == float(hdf_dicts[i]['_FillValue']))
makenan = True
except KeyError:
makenan = False
try:
scale = float(hdf_dicts[i]['scale_factor'])
offset = float(hdf_dicts[i]['add_offset'])
# print 'MODIS array: %s, type: %s' % (i, modis[i].dtype)
if scale.is_integer():
scale = int(scale)
makenan = False
if scale != 1 and offset == 0:
hdf[i] = hdf[i] * scale + offset
except:
if issubclass(hdf[i].dtype.type, np.integer):
makenan = False
if makenan:
hdf[i][bad_points] = np.nan
if verbose:
progress(
float(tuple(i[0]
for i in values).index(i)) / len(values) * 100.)
if verbose:
endprogress()
print hdf.keys()
del datsds, sds, datsub
return hdf, hdf_dicts
def load_emas(datfile):
"""
Name:
load_emas
Purpose:
to compile functions required to load emas files
from within another script.
Similar to load_modis
Calling Sequence:
emas,emas_dict = load_emas(datfile)
Input:
datfile name (hdf files)
Output:
emas dictionary with tau, ref, etau, eref, phase, qa
emas_dicts : metadate for each of the variables
Keywords:
none
Dependencies:
gdal
numpy
gc: for clearing the garbage
Required files:
dat files
Example:
...
Modification History:
Written (v1.0): Samuel LeBlanc, 2014-12-08, NASA Ames
"""
import numpy as np
from osgeo import gdal
from Sp_parameters import startprogress, progress, endprogress
emas_values = ( #('cloud_top',57),
('phase', 53),
# ('cloud_top_temp',58),
('ref', 66),
('tau', 72),
# ('cwp',82),
('eref', 90),
('etau', 93),
# ('ecwp',96),
('multi_layer', 105),
('qa', 123),
# ('cloud_mask',110)
)
datsds = gdal.Open(datfile)
datsub = datsds.GetSubDatasets()
print 'Outputting the Data subdatasets:'
for i in range(len(datsub)):
if any(i in val for val in emas_values):
print '\x1b[1;36m%i: %s\x1b[0m' % (i, datsub[i][1])
else:
print str(i) + ': ' + datsub[i][1]
emas = dict()
meta = datsds.GetMetadata()
import gc
gc.collect()
emas_dicts = dict()
startprogress('Running through modis values')
for i, j in emas_values:
sds = gdal.Open(datsub[j][0])
emas_dicts[i] = sds.GetMetadata()
emas[i] = np.array(sds.ReadAsArray())
makenan = True
bad_points = np.where(emas[i] == float(emas_dicts[i]['_FillValue']))
try:
scale = float(emas_dicts[i]['scale_factor'])
offset = float(emas_dicts[i]['add_offset'])
# print 'MODIS array: %s, type: %s' % (i, modis[i].dtype)
if scale.is_integer():
scale = int(scale)
makenan = False
if scale != 1 and offset == 0:
emas[i] = emas[i] * scale + offset
except:
if issubclass(emas[i].dtype.type, np.integer):
makenan = False
if makenan:
emas[i][bad_points] = np.nan
progress(
float(tuple(i[0] for i in emas_values).index(i)) /
len(emas_values) * 100.)
endprogress()
print emas.keys()
del datsds, sds, datsub
return emas, emas_dicts
from Sp_parameters import startprogress, progress, endprogress
import gc
gc.collect()
# <codecell>
tuple(i[0] for i in modis_values).index('etau')
# <codecell>
modis = dict()
modis_dicts = dict()
startprogress('Running through modis values')
for i, j in modis_values:
sds = gdal.Open(myd_dat_sub[j][0])
modis_dicts[i] = sds.GetMetadata()
modis[i] = np.array(sds.ReadAsArray()) * float(
modis_dicts[i]['scale_factor']) + float(
modis_dicts[i]['add_offset'])
modis[i][modis[i] == float(modis_dicts[i]['_FillValue'])] = np.nan
progress(
float(tuple(i[0] for i in modis_values).index(i)) /
len(modis_values) * 100.)
endprogress()
# <codecell>
print modis.keys()
print modis_dicts.keys()
def load_hdf(datfile,values=None,verbose=True,all_values=False):
"""
Name:
load_hdf
Purpose:
to compile functions required to load emas files
from within another script.
Similar to load_modis
Calling Sequence:
hdf_dat,hdf_dict = load_hdf(datfile,Values=None,verbose=True,all_values=False)
Input:
datfile name (hdf files)
Output:
hdf_dat dictionary with the names of values saved, with associated dictionary values
hdf_dicts : metadate for each of the variables
Keywords:
values: if ommitted, only outputs the names of the variables in file
needs to be a tuple of 2 element tuples (first element name of variable, second number of record)
example: modis_values=(('cloud_top',57),('phase',53),('cloud_top_temp',58),('ref',66),('tau',72))
verbose: if true (default), then everything is printed. if false, nothing is printed
all_values: if True, then outputs all the values with their original names, (defaults to False), overrides values keyword
Dependencies:
gdal
numpy
gc: for clearing the garbage
Sp_parameters for progress issuer
Required files:
dat files
Example:
...
Modification History:
Written (v1.0): Samuel LeBlanc, 2014-12-10, NASA Ames
Modified (v1.1): Samuel LeBlanc, 2015-04-10, NASA Ames
- added verbose keyword
Modified (v1.2): Samuel LeBlanc, 2016-05-07, Osan AFB, Korea
- added error handling for missing fill value
Modified (v1.3): Samuel LeBlanc, 2016-11-15, NASA Ames
- added all_values keyword
"""
import numpy as np
from osgeo import gdal
from Sp_parameters import startprogress, progress, endprogress
datsds = gdal.Open(datfile)
datsub = datsds.GetSubDatasets()
if verbose:
print 'Outputting the Data subdatasets:'
for i in range(len(datsub)):
if values:
if any(i in val for val in values):
print '\x1b[1;36m%i: %s\x1b[0m' %(i,datsub[i][1])
else:
print str(i)+': '+datsub[i][1]
else:
print str(i)+': '+datsub[i][1]
if all_values:
values = []
for i in range(len(datsub)):
values.append((datsub[i][1].split(' ')[1],i))
values = tuple(values)
if not values:
if verbose:
print 'Done going through file... Please supply pairs of name and index for reading file'
print " in format values = (('name1',index1),('name2',index2),('name3',index3),...)"
print " where namei is the nameof the returned variable, and indexi is the index of the variable (from above)"
return None, None
hdf = dict()
meta = datsds.GetMetadata()
import gc; gc.collect()
hdf_dicts = dict()
if verbose:
startprogress('Running through data values')
for i,j in values:
sds = gdal.Open(datsub[j][0])
hdf_dicts[i] = sds.GetMetadata()
hdf[i] = np.array(sds.ReadAsArray())
if not hdf[i].any():
import pdb; pdb.set_trace()
try:
bad_points = np.where(hdf[i] == float(hdf_dicts[i]['_FillValue']))
makenan = True
except KeyError:
makenan = False
except ValueError:
makenan = False
print '*** FillValue not used to replace NANs, will have to do manually ***'
try:
scale = float(hdf_dicts[i]['scale_factor'])
offset = float(hdf_dicts[i]['add_offset'])
# print 'MODIS array: %s, type: %s' % (i, modis[i].dtype)
if scale.is_integer():
scale = int(scale)
makenan = False
if scale != 1 and offset == 0:
hdf[i] = hdf[i]*scale+offset
except:
if issubclass(hdf[i].dtype.type, np.integer):
makenan = False
if makenan:
hdf[i][bad_points] = np.nan
if verbose:
progress(float(tuple(i[0] for i in values).index(i))/len(values)*100.)
if verbose:
endprogress()
print hdf.keys()
del datsds,sds,datsub
return hdf,hdf_dicts
def load_modis(geofile,datfile):
"""
Name:
load_modis
Purpose:
to compile functions required to load Modis files
from within another script
Calling Sequence:
modis,modis_dict = load_modis(geofile,datfile)
Input:
geofile name
datfile name (hdf files)
Output:
modis dictionary with tau, ref, etau, eref, phase, qa
modis_dicts : metadate for each of the variables
Keywords:
none
Dependencies:
gdal
numpy
gc: for clearing the garbage
Required files:
geo and dat files
Example:
...
Modification History:
Written (v1.0): Samuel LeBlanc, 2014-12-08, NASA Ames
"""
import numpy as np
from osgeo import gdal
from Sp_parameters import startprogress, progress, endprogress
modis_values = (#('cloud_top',57),
('phase',53),
# ('cloud_top_temp',58),
('ref',66),
('tau',72),
# ('cwp',82),
('eref',90),
('etau',93),
# ('ecwp',96),
('multi_layer',105),
('qa',123),
('cth',183)
# ('cloud_mask',110)
)
geosds = gdal.Open(geofile)
datsds = gdal.Open(datfile)
geosub = geosds.GetSubDatasets()
datsub = datsds.GetSubDatasets()
print 'Outputting the Geo subdatasets:'
for i in range(len(geosub)):
print str(i)+': '+geosub[i][1]
print 'Outputting the Data subdatasets:'
for i in range(len(datsub)):
if any(i in val for val in modis_values):
print '\x1b[1;36m%i: %s\x1b[0m' %(i,datsub[i][1])
else:
print str(i)+': '+datsub[i][1]
latsds = gdal.Open(geosub[12][0],gdal.GA_ReadOnly)
lonsds = gdal.Open(geosub[13][0],gdal.GA_ReadOnly)
szasds = gdal.Open(geosub[21][0],gdal.GA_ReadOnly)
modis = dict()
modis['lat'] = latsds.ReadAsArray()
modis['lon'] = lonsds.ReadAsArray()
modis['sza'] = szasds.ReadAsArray()
print modis['lon'].shape
meta = datsds.GetMetadata()
import gc; gc.collect()
modis_dicts = dict()
startprogress('Running through modis values')
for i,j in modis_values:
sds = gdal.Open(datsub[j][0])
modis_dicts[i] = sds.GetMetadata()
modis[i] = np.array(sds.ReadAsArray())
makenan = True
bad_points = np.where(modis[i] == float(modis_dicts[i]['_FillValue']))
scale = float(modis_dicts[i]['scale_factor'])
offset = float(modis_dicts[i]['add_offset'])
# print 'MODIS array: %s, type: %s' % (i, modis[i].dtype)
if scale.is_integer():
scale = int(scale)
makenan = False
if scale != 1 and offset == 0:
modis[i] = modis[i]*scale+offset
if makenan:
modis[i][bad_points] = np.nan
progress(float(tuple(i[0] for i in modis_values).index(i))/len(modis_values)*100.)
endprogress()
print modis.keys()
del geosds, datsds,sds,lonsds,latsds,geosub,datsub
return modis,modis_dicts
if __name__ == "__main__":
gdal.Open(myd_dat_sub[53][0]).GetMetadata()
mm = dict()
mm['one'] = gdal.Open(myd_dat_sub[72][0]).GetMetadata()
mm['two'] = gdal.Open(myd_dat_sub[74][0]).GetMetadata()
mm['two']['_FillValue']
from Sp_parameters import startprogress, progress, endprogress
import gc; gc.collect()
tuple(i[0] for i in modis_values).index('etau')
modis = dict()
modis_dicts = dict()
startprogress('Running through modis values')
for i,j in modis_values:
sds = gdal.Open(myd_dat_sub[j][0])
modis_dicts[i] = sds.GetMetadata()
modis[i] = np.array(sds.ReadAsArray())*float(modis_dicts[i]['scale_factor'])+float(modis_dicts[i]['add_offset'])
modis[i][modis[i] == float(modis_dicts[i]['_FillValue'])] = np.nan
progress(float(tuple(i[0] for i in modis_values).index(i))/len(modis_values)*100.)
endprogress()
print modis.keys()
print modis_dicts.keys()
def Prep_DARE_single_sol(fname,f_calipso,fp_rtm,fp_fuliou,fp_alb=None,surface_type='ocean',vv='v1',verbose=True):
"""
Purpose:
Main function to create the files for the DARE calculations for a single solx file defined by fname
Input:
fname: full file path for matlab file solution for single solutions
(e.g., MOCsolutions20150508T183717_19374_x20080x2D070x2D11120x3A250x2CPoint0x2313387030x2F25645720x2CH0x.mat)
f_calipso: full file path for Calipso file
fp_rtm: filepath for baseline rtm folder (subset of it will have input, output)
fp_fuliou: full filepath for fuliou executable
Output:
input files for fuliou
list file for calling fuliou from NAS
Keywords:
fp_alb: File path for MODIS surface albedo, must be defined if surfacetype is land_MODIS
surface_type: (default to ocean) can be either ocean, land, or land_MODIS
vv: (default to v1) version number
Dependencies:
numpy
Run_fuliou (this file)
os
scipy
datetime
Sp_parameters
Required files:
matlab MOC solution for single solx
Calipso matlab file
yohei_MOC_lambda.mat file
Example:
...
Modification History:
Written (v1.0): Samuel LeBlanc, 2017-03-17, Santa Cruz, CA
migrated to python from matlab based on codes in read_FuLiou_input_file_Calipso_revFeb2015.m
originally written by John Livingston
"""
import numpy as np
import scipy.io as sio
import os
from datetime import datetime
from Run_fuliou import get_MODIS_surf_albedo
from Sp_parameters import startprogress, progress, endprogress
# load the materials
sol = sio.loadmat(fname)
name = fname.split(os.path.sep)[-1]
da,num,xt = name.split('_')
num = int(num)-1 #convert from matlab indexing to python
da = da.lstrip('MOCsolutions')
xt = xt.rstrip('.mat')
if verbose: print 'Preparing the input files for pixel: {}'.format(num)
lm = sio.loadmat(fp_rtm+'yohei_MOC_lambda.mat')
cal = sio.loadmat(f_calipso)
# prep the write out paths
fp_in = os.path.join(fp_rtm,'input','MOC_1solx_DARE_{vv}_{num}'.format(vv=vv,num=num))
fp_out = os.path.join(fp_rtm,'output','MOC_1solx_DARE_{vv}_{num}'.format(vv=vv,num=num))
if not os.path.exists(fp_in):
os.makedirs(fp_in)
if not os.path.exists(fp_out):
os.makedirs(fp_out)
f_list = open(os.path.join(fp_rtm,'run','list_MOC_single_solx_DARE_{vv}_{num}.sh'.format(vv=vv,num=num)),'w')
print f_list.name
# prep the standard input definitions
tt = datetime.strptime(cal['calipso_date'][num],'%Y-%m-%d')
geo = {'lat':cal['calipso_lat_oneday'][num], 'lon':cal['calipso_lon_oneday'][num],
'utc':cal['calipso_time_oneday'][num][0]/100.0,'year':tt.year,'month':tt.month,'day':tt.day}
aero = {'wvl_arr':sol['solutions']['lambda'][0,0][0]*1000.0,
'z_arr':[cal['calipso_zmin_oneday'][num][0],cal['calipso_zmax_oneday'][num][0]]}
if surface_type=='ocean':
albedo = {'sea_surface_albedo':True,'land_surface_albedo':False,'modis_surface_albedo':False}
elif surface_type=='land':
albedo = {'sea_surface_albedo':False,'land_surface_albedo':True,'modis_surface_albedo':False}
elif surface_type=='land_MODIS':
albedo = {'sea_surface_albedo':False,'land_surface_albedo':True,'modis_surface_albedo':False}
if not fp_alb:
raise ValueError('fp_alb is not set, must be set to find the MCD43GF files')
albedo['modis_albedo'] = get_MODIS_surf_albedo(fp_alb,tt.timetuple().tm_yday,geo['lat'],geo['lon'],year_of_MODIS=2007)
else:
raise ValueError("surface_type can only be 'ocean', 'land', or 'land_MODIS'")
# Run through the possible solutions to the files and write out the fuliou input files
startprogress('Writing MOC files')
for i in xrange(len(sol['ssa'])):
input_file = os.path.join(fp_in,'MOC_{num}_{i:04d}.datin'.format(num=num,i=i))
output_file = os.path.join(fp_out,'MOC_{num}_{i:04d}.wrt'.format(num=num,i=i))
aero['ssa'] = np.array([sol['ssa'][i,:],sol['ssa'][i,:]])
aero['asy'] = np.array([sol['asym'][i,:],sol['asym'][i,:]])
aero['ext'] = np.array([sol['ext'][i,:],sol['ext'][i,:]])
write_fuliou_input(input_file,geo=geo,aero=aero,albedo=albedo,verbose=False)
progress(float(i)/float(len(sol['ssa']))*100.0)
f_list.write(fp_fuliou+' '+input_file+' '+output_file+'\n')
endprogress()
sel = sol['solutions']['select'][0,0]
i_str = ['m1','s0','p1']
for ie in [-1,0,1]:
for ia in [-1,0,1]:
for im in [-1,0,1]:
form = {'num':num,'e':i_str[ie+1],'a':i_str[ia+1],'s':i_str[im+1]}
input_file = os.path.join(fp_in,'MOC_{num}_{e}{a}{s}.datin'.format(**form))
output_file = os.path.join(fp_out,'MOC_{num}_{e}{a}{s}.wrt'.format(**form))
aero['ssa'] = np.array([sel['ssa'][0,0][0,:]+ia*sel['ssa'][0,0][1,:]]*2)
aero['asy'] = np.array([sel['asym'][0,0][0,:]+im*sel['asym'][0,0][1,:]]*2)
aero['ext'] = np.array([sel['ext'][0,0][0,:]+ie*sel['ext'][0,0][1,:]]*2)
write_fuliou_input(input_file,geo=geo,aero=aero,albedo=albedo,verbose=False)
print '{e}{a}{s}'.format(**form)
f_list.write(fp_fuliou+' '+input_file+' '+output_file+'\n')
f_list.close()