def test_5_to_1(self):
"""test the 5km to 1km interpolation facility
"""
#gfilename = "testdata/MOD03_A12097_174256_2012097175435.hdf"
gfilename = "/san1/test/data/modis/MOD03_A12097_174256_2012097175435.hdf"
#filename = "testdata/MOD021km_A12097_174256_2012097175435.hdf"
filename = "/san1/test/data/modis/MOD021km_A12097_174256_2012097175435.hdf"
from pyhdf.SD import SD
from pyhdf.error import HDF4Error
try:
gdata = SD(gfilename)
data = SD(filename)
except HDF4Error:
print "Failed reading both eos-hdf files %s and %s" % (gfilename, filename)
return
glats = gdata.select("Latitude")[:]
glons = gdata.select("Longitude")[:]
lats = data.select("Latitude")[:]
lons = data.select("Longitude")[:]
tlons, tlats = modis5kmto1km(lons, lats)
self.assert_(np.allclose(tlons, glons, atol=0.05))
self.assert_(np.allclose(tlats, glats, atol=0.05))
def get_lat_lon_modis(satscene, options):
"""Read lat and lon.
"""
filename_tmpl = satscene.time_slot.strftime(options["geofile"])
file_list = glob.glob(os.path.join(options["dir"], filename_tmpl))
if len(file_list) == 0:
# Try in the same directory as the data
data_dir = os.path.split(options["filename"])[0]
file_list = glob.glob(os.path.join(data_dir, filename_tmpl))
if len(file_list) > 1:
logger.warning("More than 1 geolocation file matching!")
filename = max(file_list, key=lambda x: os.stat(x).st_mtime)
coarse_resolution = 1000
elif len(file_list) == 0:
logger.warning("No geolocation file matching " + filename_tmpl
+ " in " + options["dir"])
logger.debug("Using 5km geolocation and interpolating")
filename = options["filename"]
coarse_resolution = 5000
else:
filename = file_list[0]
coarse_resolution = 1000
logger.debug("Loading geolocation file: " + str(filename)
+ " at resolution " + str(coarse_resolution))
resolution = options["resolution"]
data = SD(str(filename))
lat = data.select("Latitude")
fill_value = lat.attributes()["_FillValue"]
lat = np.ma.masked_equal(lat.get(), fill_value)
lon = data.select("Longitude")
fill_value = lon.attributes()["_FillValue"]
lon = np.ma.masked_equal(lon.get(), fill_value)
if resolution == coarse_resolution:
return lat, lon
cores = options["cores"]
from geotiepoints import modis5kmto1km, modis1kmto500m, modis1kmto250m
logger.debug("Interpolating from " + str(coarse_resolution)
+ " to " + str(resolution))
if coarse_resolution == 5000:
lon, lat = modis5kmto1km(lon, lat)
if resolution == 500:
lon, lat = modis1kmto500m(lon, lat, cores)
if resolution == 250:
lon, lat = modis1kmto250m(lon, lat, cores)
return lat, lon
def get_lat_lon_modis(satscene, options):
"""Read lat and lon.
"""
filename_tmpl = satscene.time_slot.strftime(options["geofile"])
file_list = glob.glob(os.path.join(options["dir"], filename_tmpl))
if len(file_list) == 0:
# Try in the same directory as the data
data_dir = os.path.split(options["filename"])[0]
file_list = glob.glob(os.path.join(data_dir, filename_tmpl))
if len(file_list) > 1:
logger.warning("More than 1 geolocation file matching!")
filename = max(file_list, key=lambda x: os.stat(x).st_mtime)
coarse_resolution = 1000
elif len(file_list) == 0:
logger.warning("No geolocation file matching " + filename_tmpl
+ " in " + options["dir"])
logger.debug("Using 5km geolocation and interpolating")
filename = options["filename"]
coarse_resolution = 5000
else:
filename = file_list[0]
coarse_resolution = 1000
logger.debug("Loading geolocation file: " + str(filename)
+ " at resolution " + str(coarse_resolution))
resolution = options["resolution"]
data = SD(str(filename))
lat = data.select("Latitude")
fill_value = lat.attributes()["_FillValue"]
lat = np.ma.masked_equal(lat.get(), fill_value)
lon = data.select("Longitude")
fill_value = lon.attributes()["_FillValue"]
lon = np.ma.masked_equal(lon.get(), fill_value)
if resolution == coarse_resolution:
return lat, lon
cores = options["cores"]
from geotiepoints import modis5kmto1km, modis1kmto500m, modis1kmto250m
logger.debug("Interpolating from " + str(coarse_resolution)
+ " to " + str(resolution))
if coarse_resolution == 5000:
lon, lat = modis5kmto1km(lon, lat)
if resolution == 500:
lon, lat = modis1kmto500m(lon, lat, cores)
if resolution == 250:
lon, lat = modis1kmto250m(lon, lat, cores)
return lat, lon
def test_5_to_1(self):
"""test the 5km to 1km interpolation facility
"""
with h5py.File(FILENAME_FULL) as h5f:
glons = h5f['longitude'][:] / 1000.
glats = h5f['latitude'][:] / 1000.
with h5py.File(FILENAME_5KM) as h5f:
lons = h5f['longitude'][:] / 1000.
lats = h5f['latitude'][:] / 1000.
tlons, tlats = modis5kmto1km(lons, lats)
self.assert_(np.allclose(tlons, glons, atol=0.05))
self.assert_(np.allclose(tlats, glats, atol=0.05))
def test_5_to_1(self):
"""test the 5km to 1km interpolation facility
"""
with h5py.File(FILENAME_FULL) as h5f:
glons = h5f['longitude'][:] / 1000.
glats = h5f['latitude'][:] / 1000.
with h5py.File(FILENAME_5KM) as h5f:
lons = h5f['longitude'][:] / 1000.
lats = h5f['latitude'][:] / 1000.
tlons, tlats = modis5kmto1km(lons, lats)
self.assert_(np.allclose(tlons, glons, atol=0.05))
self.assert_(np.allclose(tlats, glats, atol=0.05))
def _create_coord_list(self, filenames, variable=None):
import datetime as dt
from cis.time_util import convert_time_since_to_std_time, cis_standard_time_unit
from cis.utils import concatenate
from cf_units import Unit
from geotiepoints import modis5kmto1km
variables = ['Latitude', 'Longitude', 'View_time']
logging.info("Listing coordinates: " + str(variables))
sdata, vdata = hdf.read(filenames, variables)
apply_interpolation = False
if variable is not None:
scale = self.__get_data_scale(filenames[0], variable)
apply_interpolation = True if scale is "1km" else False
lat_data = hdf.read_data(sdata['Latitude'], _get_MODIS_SDS_data)
lat_metadata = hdf.read_metadata(sdata['Latitude'], "SD")
lon_data = hdf.read_data(sdata['Longitude'], _get_MODIS_SDS_data)
lon_metadata = hdf.read_metadata(sdata['Longitude'], "SD")
if apply_interpolation:
lon_data, lat_data = modis5kmto1km(lon_data, lat_data)
lat_coord = Coord(lat_data, lat_metadata, 'Y')
lon_coord = Coord(lon_data, lon_metadata, 'X')
time = sdata['View_time']
time_metadata = hdf.read_metadata(time, "SD")
# Ensure the standard name is set
time_metadata.standard_name = 'time'
time_metadata.units = cis_standard_time_unit
t_arrays = []
for f, d in zip(filenames, time):
time_start = self._get_start_date(f)
t_data = _get_MODIS_SDS_data(
d) / 24.0 # Convert hours since to days since
t_offset = time_start - dt.datetime(1600, 1,
1) # Convert to CIS time
t_arrays.append(t_data + t_offset.days)
time_coord = Coord(concatenate(t_arrays), time_metadata, "T")
return CoordList([lat_coord, lon_coord, time_coord])
def _create_coord_list(self, filenames, variable=None):
import datetime as dt
from cis.time_util import convert_time_since_to_std_time, cis_standard_time_unit
from cis.utils import concatenate
from cf_units import Unit
from geotiepoints import modis5kmto1km
variables = ['Latitude', 'Longitude', 'View_time']
logging.info("Listing coordinates: " + str(variables))
sdata, vdata = hdf.read(filenames, variables)
apply_interpolation = False
if variable is not None:
scale = self.__get_data_scale(filenames[0], variable)
apply_interpolation = True if scale is "1km" else False
lat_data = hdf.read_data(sdata['Latitude'], _get_MODIS_SDS_data)
lat_metadata = hdf.read_metadata(sdata['Latitude'], "SD")
lon_data = hdf.read_data(sdata['Longitude'], _get_MODIS_SDS_data)
lon_metadata = hdf.read_metadata(sdata['Longitude'], "SD")
if apply_interpolation:
lon_data, lat_data = modis5kmto1km(lon_data, lat_data)
lat_coord = Coord(lat_data, lat_metadata, 'Y')
lon_coord = Coord(lon_data, lon_metadata, 'X')
time = sdata['View_time']
time_metadata = hdf.read_metadata(time, "SD")
# Ensure the standard name is set
time_metadata.standard_name = 'time'
time_metadata.units = cis_standard_time_unit
t_arrays = []
for f, d in zip(filenames, time):
time_start = self._get_start_date(f)
t_data = _get_MODIS_SDS_data(d) / 24.0 # Convert hours since to days since
t_offset = time_start - dt.datetime(1600, 1, 1) # Convert to CIS time
t_arrays.append(t_data + t_offset.days)
time_coord = Coord(concatenate(t_arrays), time_metadata, "T")
return CoordList([lat_coord, lon_coord, time_coord])
def get_lonlat(self, resolution, cores=1):
"""Read lat and lon.
"""
if resolution in self.areas:
return self.areas[resolution]
logger.debug("generating lon, lat at %d", resolution)
if self.geofile is not None:
coarse_resolution = 1000
filename = self.geofile
else:
coarse_resolution = 5000
logger.info("Using 5km geolocation and interpolating")
filename = (self.datafiles.get(1000) or
self.datafiles.get(500) or
self.datafiles.get(250))
logger.debug("Loading geolocation from file: " + str(filename)
+ " at resolution " + str(coarse_resolution))
data = SD(str(filename))
lat = data.select("Latitude")
fill_value = lat.attributes()["_FillValue"]
lat = np.ma.masked_equal(lat.get(), fill_value)
lon = data.select("Longitude")
fill_value = lon.attributes()["_FillValue"]
lon = np.ma.masked_equal(lon.get(), fill_value)
if resolution == coarse_resolution:
self.areas[resolution] = lon, lat
return lon, lat
from geotiepoints import modis5kmto1km, modis1kmto500m, modis1kmto250m
logger.debug("Interpolating from " + str(coarse_resolution)
+ " to " + str(resolution))
if coarse_resolution == 5000:
lon, lat = modis5kmto1km(lon, lat)
if resolution == 500:
lon, lat = modis1kmto500m(lon, lat, cores)
if resolution == 250:
lon, lat = modis1kmto250m(lon, lat, cores)
self.areas[resolution] = lon, lat
return lon, lat
def get_lonlat(self, resolution, time_slot, cores=1):
"""Read lat and lon.
"""
if (resolution, time_slot) in self.areas:
return self.areas[resolution, time_slot]
logger.debug("generating lon, lat at %d", resolution)
if self.geofile is not None:
coarse_resolution = 1000
filename = self.geofile
else:
coarse_resolution = 5000
logger.info("Using 5km geolocation and interpolating")
filename = (self.datafiles.get(1000) or
self.datafiles.get(500) or
self.datafiles.get(250))
logger.debug("Loading geolocation from file: " + str(filename)
+ " at resolution " + str(coarse_resolution))
data = SD(str(filename))
lat = data.select("Latitude")
fill_value = lat.attributes()["_FillValue"]
lat = np.ma.masked_equal(lat.get(), fill_value)
lon = data.select("Longitude")
fill_value = lon.attributes()["_FillValue"]
lon = np.ma.masked_equal(lon.get(), fill_value)
if resolution == coarse_resolution:
self.areas[resolution] = lon, lat
return lon, lat
from geotiepoints import modis5kmto1km, modis1kmto500m, modis1kmto250m
logger.debug("Interpolating from " + str(coarse_resolution)
+ " to " + str(resolution))
if coarse_resolution == 5000:
lon, lat = modis5kmto1km(lon, lat)
if resolution == 500:
lon, lat = modis1kmto500m(lon, lat, cores)
if resolution == 250:
lon, lat = modis1kmto250m(lon, lat, cores)
self.areas[resolution, time_slot] = lon, lat
return lon, lat
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
"""
import h5py
import numpy as np
from geotiepoints import modis5kmto1km
FILENAME_FULL = 'testdata/test_5_to_1_geoloc_full.h5'
FILENAME_5KM = 'testdata/test_5_to_1_geoloc_5km.h5'
with h5py.File(FILENAME_FULL) as h5f:
glons = h5f['longitude'][:] / 1000.
glats = h5f['latitude'][:] / 1000.
with h5py.File(FILENAME_5KM) as h5f:
lons = h5f['longitude'][:] / 1000.
lats = h5f['latitude'][:] / 1000.
tlons, tlats = modis5kmto1km(lons, lats)
print np.allclose(tlons, glons, atol=0.05)
print np.allclose(tlats, glats, atol=0.05)
def get_modis_1km_rgb(filename, verbose=False):
""" get RGB reflectance from MxD021KM file.
Parameters
----------
filename : str
MxD021KM file.
Returns
-------
rgb : 3D numpy array
R, G, and B channel TOA reflectance
(without solar zenith angel correction.)
"""
varname_list = [\
'EV_250_Aggr1km_RefSB', \
'EV_500_Aggr1km_RefSB', \
'EV_500_Aggr1km_RefSB', \
]
ind_list = [0, 1, 0]
if verbose:
print(' - get_modis_1km_rgb: reading ' + filename)
# open dataset
fid = Dataset(filename, 'r')
# get RGB
rgb = []
for i in range(len(varname_list)):
varname = varname_list[i]
ind = ind_list[i]
var = fid.variables[varname]
var.set_auto_maskandscale(False)
scale_factor = var.reflectance_scales[ind]
offset = var.reflectance_offsets[ind]
var = var
var = var[ind, :, :]
var = (var - offset) * scale_factor
var[var < 0.0] = 0.0
var[var > 1.0] = 1.0
if i == 0:
rgb = np.zeros((var.shape[0], var.shape[1], 3))
rgb[:, :, i] = var
# get latitude and lonitude
lat = fid.variables['Latitude'][:]
lon = fid.variables['Longitude'][:]
lon, lat = modis5kmto1km(lon, lat)
# close dataset
fid.close()
return lat, lon, rgb
print(idx,sds)
# %% {"scrolled": true}
lat_5km = the_file.select('Latitude') # select sds
lon_5km = the_file.select('Longitude') # select sds
print(lat_5km.info())
print(help(lat_5km.info))
lat_5km=lat_5km[:,:]
lon_5km=lon_5km[:,:]
# %% [markdown]
# ## get the coase 5km lat/lons
# %%
import geotiepoints
lons_1km, lats_1km = geotiepoints.modis5kmto1km(lon_5km, lat_5km)
# %% [markdown]
# ## Interpolate to 1 km using geotiepoints
#
# Use https://python-geotiepoints.readthedocs.io/en/latest/
# %%
lons_1km.shape
# %% [markdown]
# ## open one of the datasets (EV_1KM_Emissive) and get its shape and data type
# %%
longwave_data = the_file.select('EV_1KM_Emissive') # select sds
print(longwave_data.info())
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
"""
import h5py
import numpy as np
from geotiepoints import modis5kmto1km
FILENAME_FULL = 'testdata/test_5_to_1_geoloc_full.h5'
FILENAME_5KM = 'testdata/test_5_to_1_geoloc_5km.h5'
with h5py.File(FILENAME_FULL) as h5f:
glons = h5f['longitude'][:] / 1000.
glats = h5f['latitude'][:] / 1000.
with h5py.File(FILENAME_5KM) as h5f:
lons = h5f['longitude'][:] / 1000.
lats = h5f['latitude'][:] / 1000.
tlons, tlats = modis5kmto1km(lons, lats)
print np.allclose(tlons, glons, atol=0.05)
print np.allclose(tlats, glats, atol=0.05)
