def fetch_temp(self, variable='tmax', temp_units='C'):
print('Downloading new {}.....'.format(variable))
try:
topowx = TopoWX(date=self.date, bbox=self.bounds,
target_profile=self.profile,
clip_feature=self.clip_geo, out_file=self.file_path)
var = topowx.get_data_subset(grid_conform=True, var=variable,
out_file=self.file_path,
temp_units_out=temp_units)
except ValueError:
if variable == 'tmax':
variable = 'tmmx'
elif variable == 'tmin':
variable = 'tmmn'
else:
raise AttributeError
print('TopoWX temp retrieval failed, attempting same w/ Gridmet.')
gridmet = GridMet(variable, date=self.date, bbox=self.bounds,
target_profile=self.profile, clip_feature=self.clip_geo)
var = gridmet.get_data_subset()
return var
def get_precip(self):
poly_in = self.landsat.get_tile_geometry()
poly_in = Polygon(poly_in[0]['coordinates'][0])
project = partial(
pytransform,
Proj(self.profile['crs']),
Proj(init='epsg:32612'))
for_bounds = partial(
pytransform,
Proj(self.profile['crs']),
Proj(init='epsg:4326'))
dates = self.scenes['DATE_ACQUIRED'].values
# Change the coordinate system
# The issue: the CRSs for the bounding box and for the mask are different.
# In _project, the incorrect CRS was making it throw an error.
# the fix? Inputting bounds in a unprojected CRS and
# a projected shape for masking.
poly = transform(project, poly_in)
poly_bounds = transform(for_bounds, poly_in)
poly = Polygon(poly.exterior.coords)
geometry = [mapping(poly)]
geometry[0]['crs'] = CRS({'init': 'epsg:32612'})
bounds = poly_bounds.bounds
for date in dates:
outfile = os.path.join(self.root, 'precip_{}.tif'.format(date))
if not os.path.isfile(outfile):
print("Get {}".format(outfile))
d = datetime.utcfromtimestamp(date.tolist() / 1e9) # convert to a nicer format.
bds = GeoBounds(wsen=bounds)
gm = GridMet(variable='pr', clip_feature=geometry,
bbox=bds, target_profile=self.profile, date=d)
out = gm.get_data_subset()
gm.save_raster(out, self.landsat.rasterio_geometry, outfile)
def test_agrimet_mesonet_gridmet_etr(self):
agrimet = Agrimet(station=self.fetch_site,
start_date=self.start,
end_date=self.end,
interval='daily')
formed = agrimet.fetch_met_data()
agri_etr = formed['ETRS'].values
s, e = datetime.strptime(self.start, '%Y-%m-%d'), \
datetime.strptime(self.end, '%Y-%m-%d')
gridmet = GridMet('etr', start=s, end=e, lat=self.lat, lon=self.lon)
gridmet_etr = gridmet.get_point_timeseries()
gridmet_etr = gridmet_etr.values
mco = Mesonet(self.covm_mco, start=self.start, end=self.end)
mesonet_daily = mco.mesonet_etr(lat=46.3, elevation=1000.0)
mesonet_etr = mesonet_daily['ETR'].values
plt.plot(gridmet_etr[121:273], label='gridmet')
plt.plot(mesonet_etr[121:273], label='mesonet')
plt.plot(agri_etr[121:273], label='agrimet')
plt.xlabel('GROWING SEASON DAY (MAY 01 - SEP 30)')
plt.ylabel('Tall Crop Reference ET (mm) daily')
plt.legend()
# plt.show()
saved = os.path.join(os.path.dirname(__file__),
'grid_agri_meso_fig.png')
print('saved to {}'.format(saved))
plt.savefig(saved)
ga_ratio = gridmet_etr[121:273].sum() / agri_etr[121:273].sum()
print('gridmet - agrimet ratio: {}'.format(ga_ratio))
ma_ratio = mesonet_etr[121:273].sum() / agri_etr[121:273].sum()
print('mesonet - agrimet ratio: {}'.format(ma_ratio))
def fetch_gridmet(self, variable='pet'):
gridmet = GridMet(variable, date=self.date,
bbox=self.bounds,
target_profile=self.profile,
clip_feature=self.clip_geo)
var = gridmet.get_data_subset(out_filename=self.file_path)
return var
def test_conforming_array(self):
""" Test shape of Gridmet vs. Landsat image.
:return:
"""
l8 = Landsat8(self.dir_name_LC8)
polygon = l8.get_tile_geometry()
bounds = RasterBounds(affine_transform=l8.rasterio_geometry['transform'], profile=l8.rasterio_geometry)
gridmet = GridMet(self.var, date=self.date, bbox=bounds,
target_profile=l8.rasterio_geometry, clip_feature=polygon)
pet = gridmet.get_data_subset()
shape = 1, l8.rasterio_geometry['height'], l8.rasterio_geometry['width']
self.assertEqual(pet.shape, shape)
def test_save_to_netcdf(self):
""" Test save native netcdf data from Thredds.Gridmet to disk.
:return:
"""
gridmet = GridMet(self.var, date=self.date)
out = 'data/met_test/{}-{}-{}_pet.nc'.format(self.date.year,
self.date.month,
self.date.day)
gridmet.write_netcdf(outputroot=out)
self.assertTrue(os.path.exists(out))
data = open_dataset(out)
self.assertIsInstance(data, Dataset)
os.remove(out)
def correct_pr_mesonet_gridmet(start, end, lat, lon):
s, e = datetime.strptime(start, '%Y-%m-%d'), \
datetime.strptime(end, '%Y-%m-%d')
gridmet = GridMet('pr', start=s, end=e, lat=lat, lon=lon)
gridmet_ppt = gridmet.get_point_timeseries()
gridmet_ppt = gridmet_ppt.values
mco = Mesonet(LOLO_MCO, start=start, end=end)
mesonet_ppt = mco.mesonet_ppt(daily=True)
mesonet_ppt = mesonet_ppt['Precipitation'].values
mesonet_ppt = mesonet_ppt.reshape((mesonet_ppt.shape[0], 1))
meso_gs_ppt, grid_gs_ppt = mesonet_ppt[121:273], gridmet_ppt[121:273]
comp = hstack((meso_gs_ppt, grid_gs_ppt))
pass
def test_conforming_array_local(self):
l8 = Landsat8(self.dir_name_LC8)
polygon = l8.get_tile_geometry()
bounds = RasterBounds(
affine_transform=l8.rasterio_geometry['transform'],
profile=l8.rasterio_geometry)
gridmet = GridMet(self.var,
date=self.date,
bbox=bounds,
target_profile=l8.rasterio_geometry,
clip_feature=polygon)
pet = gridmet.get_data_subset(
file_url='/home/dgketchum/Downloads/{}_{}.nc'.format(
self.var, self.date.year))
shape = 1, l8.rasterio_geometry['height'], l8.rasterio_geometry[
'width']
self.assertEqual(pet.shape, shape)
def test_conforming_array_to_native(self):
""" Test confoming array to native Gridmet raster.
Conforming array is what Thredds.Gridmet will build given geometry
paramters derived from LandsatImage object. This test builds that
array, and then compares it with several day's of native Gridmet netcdf
data. The rasters can't align perfectly, as the grid has been resampled.
This is built to look up 30 points, extract their location, get the native
raster value (i.e., geo) at that location, and the (local) conforming
array value. The ratio of the means from each raster must be w/in
0.5%.
:return:
"""
l8 = Landsat8(self.dir_name_LC8)
polygon = l8.get_tile_geometry()
bounds = RasterBounds(
affine_transform=l8.rasterio_geometry['transform'],
profile=l8.rasterio_geometry,
latlon=True)
for day in rrule(DAILY, dtstart=self.start, until=self.end):
gridmet = GridMet(self.var,
date=day,
bbox=bounds,
target_profile=l8.rasterio_geometry,
clip_feature=polygon)
date_str = datetime.strftime(day, '%Y-%m-%d')
met_arr = os.path.join(self.grimet_raster_dir,
'met_{}_{}.tif'.format(date_str, self.var))
met = gridmet.get_data_subset(out_filename=met_arr)
native = os.path.join(self.grimet_raster_dir,
'{}_pet.tif'.format(date_str))
points_dict = multi_raster_point_extract(
local_raster=met_arr,
geographic_raster=native,
points=self.scene_points,
image_profile=l8.rasterio_geometry)
geo_list, local_list = [], []
for key, val in points_dict.items():
geo_list.append(val['geo_val'])
local_list.append(val['local_val'])
ratio = mean(geo_list) / mean(local_list)
print('Ratio on {} of CONUSRaster:LocalRaster calculated is {}.'.
format(datetime.strftime(day, '%Y-%m-%d'), ratio))
self.assertAlmostEqual(ratio, 1.0, delta=0.005)
os.remove(met_arr)
def test_agrimet_gridmet_precip(self):
agrimet = Agrimet(station=self.fetch_site,
start_date=self.start,
end_date=self.end,
interval='daily')
formed = agrimet.fetch_met_data()
agri_ppt = formed['PP'].values
s, e = datetime.strptime(self.start, '%Y-%m-%d'), \
datetime.strptime(self.end, '%Y-%m-%d')
gridmet = GridMet('pr', start=s, end=e, lat=self.lat, lon=self.lon)
gridmet_ppt = gridmet.get_point_timeseries()
gridmet_ppt = gridmet_ppt.values
difference = abs(gridmet_ppt - agri_ppt)
self.assertLess(nanmean(difference), -100000)
def test_get_time_series(self):
""" Test native pet rasters vs. xarray netcdf point extract.
:return:
"""
rasters = os.listdir(self.grimet_raster_dir)
for ras in rasters:
if ras.endswith('pet.tif'):
dt = datetime.strptime(ras[:10], '%Y-%m-%d')
raster = os.path.join(self.grimet_raster_dir, ras)
points = raster_point_extract(raster, self.agri_points, dt)
for key, val in points.items():
lon, lat = val['coords']
_, var = ras.split('_')
var = var.replace('.tif', '')
gridmet = GridMet(var, date=dt, lat=lat, lon=lon)
gridmet_pet = gridmet.get_point_timeseries()
val[dt][1] = gridmet_pet.iloc[0, 0]
for key, val in points.items():
self.assertEqual(val[dt][0], val[dt][1])
def test_agrimet_gridmet_etr(self):
agrimet = Agrimet(station=self.fetch_site,
start_date=self.start,
end_date=self.end,
interval='daily')
formed = agrimet.fetch_met_data()
agri_etr = formed['ETRS'].values
s, e = datetime.strptime(self.start, '%Y-%m-%d'), \
datetime.strptime(self.end, '%Y-%m-%d')
gridmet = GridMet('etr', start=s, end=e, lat=self.lat, lon=self.lon)
gridmet_etr = gridmet.get_point_timeseries()
gridmet_etr = gridmet_etr.values
# plt.plot(gridmet_etr, label='gridmet')
# plt.plot(agri_etr, label='agrimet')
# plt.legend()
# plt.show()
# plt.close()
ratio = agri_etr.sum() / gridmet_etr.sum()
print('ratio: {}'.format(ratio))
def correct_etr_mesonet_gridmet(start, end, lat, lon):
s, e = datetime.strptime(start, '%Y-%m-%d'), \
datetime.strptime(end, '%Y-%m-%d')
gridmet = GridMet('etr', start=s, end=e, lat=lat, lon=lon)
gridmet_etr = gridmet.get_point_timeseries()
gridmet_etr = gridmet_etr.values
mco = Mesonet(LOLO_MCO, start=start, end=end)
mesonet_daily = mco.mesonet_etr(lat=46.3, elevation=1000.0)
mesonet_etr = mesonet_daily['ETR'].values
plt.plot(gridmet_etr[121:273], label='gridmet')
plt.plot(mesonet_etr[121:273], label='mesonet')
plt.xlabel('GROWING SEASON DAY (MAY 01 - SEP 30)')
plt.ylabel('Tall Crop Reference ET (mm) daily')
plt.legend()
plt.show()
ma_ratio = mesonet_etr[121:273].sum() / gridmet_etr[121:273].sum()
print('mesonet - gridmet ratio: {}'.format(ma_ratio))
ma_ratio = gridmet_etr[121:273].sum() / mesonet_etr[121:273].sum()
print('gridmet - mesonet ratio: {}'.format(ma_ratio))
def get_gridmet(self, start, end, lat, lon):
# gridmet params
gridmet = GridMet('pr', start=start, end=end, lat=lat, lon=lon)
ts_ppt = gridmet.get_point_timeseries()
m_ppt = ts_ppt.groupby(lambda x: x.month).sum().values
gridmet = GridMet('etr', start=start, end=end, lat=lat, lon=lon)
ts_etr = gridmet.get_point_timeseries()
m_etr = ts_etr.groupby(lambda x: x.month).sum().values
return m_ppt, m_etr
def test_elevation(self):
l8 = Landsat8(self.dir_name_LC8)
polygon = l8.get_tile_geometry()
bounds = RasterBounds(affine_transform=l8.rasterio_geometry['transform'], profile=l8.rasterio_geometry)
gridmet = GridMet('elev', date=self.date, bbox=bounds,
target_profile=l8.rasterio_geometry, clip_feature=polygon)
gridmet.save_raster()
gridmet.get_data_subset(os.path.join(self.grimet_raster_dir, 'elevation.tif'))
def get_monthly_gridmet(self):
self.geometry, bounds, clip = self._get_landsat_image()
assert self.geometry['crs'].data == self.project_crs
days = monthrange(self.year, self.month)[1]
s, e = datetime(self.year, self.month,
1), datetime(self.year, self.month, days)
kwargs = dict(start=s,
end=e,
bbox=bounds,
clip_feature=clip,
target_profile=self.geometry)
if self.variable == 'pr':
self.gridmet = GridMet(self.variable, **kwargs)
path = os.path.join(self.netcdf,
'{}_{}.nc'.format(self.variable, self.year))
self.arr = self.gridmet.get_area_timeseries(file_url=path,
operation='sum')
elif self.variable == 'temp':
path = os.path.join(self.netcdf, 'tmmx_{}.nc'.format(self.year))
self.gridmet = GridMet('tmmx', **kwargs)
mean_max = self.gridmet.get_area_timeseries(file_url=path,
operation='mean')
path = os.path.join(self.netcdf, 'tmmn_{}.nc'.format(self.year))
self.gridmet = GridMet('tmmn', **kwargs)
mean_min = self.gridmet.get_area_timeseries(file_url=path,
operation='mean')
self.arr = mean([mean_max, mean_min], axis=0)
elif self.variable == 'elev':
self.gridmet = GridMet(self.variable, **kwargs)
path = os.path.join(
self.netcdf, 'metdata_elevationdata.nc'.format(self.variable))
self.arr = self.gridmet.get_data_subset(file_url=path)
else:
raise NotImplementedError("choose from 'temp', 'pr', or 'elev'")
arr = self._clip_to_catchment()
return arr
def get_climate_timeseries(self):
bounds, geometry = self._get_bounds()
print(bounds, geometry)
dates = self.scenes['DATE_ACQUIRED'].values
all_dates = arange(datetime(self.year, 3, 1),
max(dates) + 1,
timedelta(days=1)).astype(datetime64)
for target in self.climate_targets:
out_arr = None
first = True
last = None
check = [
os.path.isfile(
os.path.join(self.root, 'climate_rasters',
'{}_{}.tif'.format(q, target))) for q in dates
]
if False in check:
for date in all_dates:
d = datetime.utcfromtimestamp(
date.tolist() / 1e9) # convert to a nicer format.
bds = GeoBounds(wsen=bounds)
gm = GridMet(variable=target,
clip_feature=geometry,
bbox=bds,
target_profile=self.profile,
date=d)
out = gm.get_data_subset_nonconform()
if first:
out_arr = zeros(out.shape)
first = False
out_arr += out
if date in dates:
out_dir = 'climate_rasters'
out_dir = os.path.join(self.root, out_dir)
if not os.path.isdir(out_dir):
os.mkdir(out_dir)
outfile = os.path.join(
out_dir, '{}_{}.tif'.format(date, target))
print("Saving {}".format(outfile))
out_final = gm.conform(out_arr)
gm.save_raster(out_final,
self.landsat.rasterio_geometry, outfile)
rmtree(gm.temp_dir)
class CatchmetGridmet():
def __init__(self, year, month, _id, variable='pr'):
home = os.path.expanduser('~')
self.netcdf = os.path.join(home, 'IrrigationGIS', 'lolo',
'gridmet_ncdf')
self.tif = os.path.join(home, 'IrrigationGIS', 'lolo', 'tif')
self.landsat = os.path.join(home, 'IrrigationGIS', 'lolo',
'LC80410282016158LGN01')
self.catchments = os.path.join(
home, 'IrrigationGIS', 'lolo', 'shp',
'Lolo_WB_Model_Calibration_Catchments_32611.shp')
self.project_crs = {'init': 'epsg:32611'}
self.year = year
self.month = month
self.id = _id
self.variable = variable
self.tif_name = os.path.join(
self.tif, 'F{}_{}_{}_{}.tif'.format(_id, variable, year, month))
self.clip_name = os.path.join(
self.tif, 'F{}_{}_{}_{}_clip.tif'.format(_id, variable, year,
month))
self.catchment = self._get_feature()
self.arr = None
self.geometry = None
self.gridmet = None
def _get_feature(self):
with fiona_open(self.catchments, 'r') as src:
assert src.crs == self.project_crs
for f in src:
if f['properties']['Id'] == self.id:
catchment = Polygon(f['geometry']['coordinates'][0])
return catchment
raise NotImplementedError()
def _get_landsat_image(self):
l8 = Landsat8(self.landsat)
bounds = RasterBounds(
affine_transform=l8.rasterio_geometry['transform'],
profile=l8.rasterio_geometry)
proj_bounds = bounds.to_epsg(32611)
clip = proj_bounds.get_shapely_polygon()
return l8.rasterio_geometry, bounds, clip
def _clip_to_catchment(self):
self.gridmet.save_raster(self.arr, self.geometry, self.tif_name)
with raster_open(self.tif_name) as src:
out_arr, out_trans = mask(src, [self.catchment],
crop=True,
all_touched=True)
out_meta = src.meta.copy()
out_meta.update({
'driver': 'GTiff',
'height': out_arr.shape[1],
'width': out_arr.shape[2],
'transform': out_trans
})
with raster_open(self.clip_name, 'w', **out_meta) as dst:
dst.write(out_arr)
return out_arr
def get_monthly_gridmet(self):
self.geometry, bounds, clip = self._get_landsat_image()
assert self.geometry['crs'].data == self.project_crs
days = monthrange(self.year, self.month)[1]
s, e = datetime(self.year, self.month,
1), datetime(self.year, self.month, days)
kwargs = dict(start=s,
end=e,
bbox=bounds,
clip_feature=clip,
target_profile=self.geometry)
if self.variable == 'pr':
self.gridmet = GridMet(self.variable, **kwargs)
path = os.path.join(self.netcdf,
'{}_{}.nc'.format(self.variable, self.year))
self.arr = self.gridmet.get_area_timeseries(file_url=path,
operation='sum')
elif self.variable == 'temp':
path = os.path.join(self.netcdf, 'tmmx_{}.nc'.format(self.year))
self.gridmet = GridMet('tmmx', **kwargs)
mean_max = self.gridmet.get_area_timeseries(file_url=path,
operation='mean')
path = os.path.join(self.netcdf, 'tmmn_{}.nc'.format(self.year))
self.gridmet = GridMet('tmmn', **kwargs)
mean_min = self.gridmet.get_area_timeseries(file_url=path,
operation='mean')
self.arr = mean([mean_max, mean_min], axis=0)
elif self.variable == 'elev':
self.gridmet = GridMet(self.variable, **kwargs)
path = os.path.join(
self.netcdf, 'metdata_elevationdata.nc'.format(self.variable))
self.arr = self.gridmet.get_data_subset(file_url=path)
else:
raise NotImplementedError("choose from 'temp', 'pr', or 'elev'")
arr = self._clip_to_catchment()
return arr
def test_instantiate(self):
""" Test instantiation of Thredds.Grimet object.
:return:
"""
gridmet = GridMet(self.var, start=self.start, end=self.end)
self.assertIsInstance(gridmet, GridMet)